(core) updates from grist-core

app/server/lib/ActiveDoc.ts

@@ -1644,7 +1644,17 @@ export class ActiveDoc extends EventEmitter {
       const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds];
       await this.applyUserActions(makeExceptionalDocSession('system'), [action]);
     }
+    try {
       await this.docStorage.removeUnusedAttachments();
+    } catch (e) {
+      // If document doesn't have _gristsys_Files, don't worry about it;
+      // if this is an error it will have already been reported, and the
+      // document can be in this state when updating initial SQL code after
+      // a schema change.
+      if (!String(e).match(/no such table: _gristsys_Files/)) {
+        throw e;
+      }
+    }
   }
 
   // Needed for test/server/migrations.js tests

buildtools/update_schema.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Regenerates typescript files with schema and sql for grist documents.
+# This needs to run whenever the document schema is changed in the data
+# engine, maintained in python code. It propagates the schema information
+# to a typescript file, and updates SQL code for initializing new documents.
+#
+# To preview what it will do, call as:
+#   buildtools/update_schema.sh schema.ts sql.ts
+# This will put schema.ts and sql.ts files in your working directory.
+# Run without any arguments to modify application files.
+#   buildtools/update_schema.sh
+# (you can see the differences with git diff if in a git repository).
+
+set -e
+
+schema_ts=$1
+sql_ts=$2
+if [[ -z "$schema_ts" ]]; then
+  # Default to regenerating regular suspects.
+  schema_ts=app/common/schema.ts
+  sql_ts=app/server/lib/initialDocSql.ts
+fi
+if [[ -z "$sql_ts" ]]; then
+  echo "Need both a schema and sql target"
+  exit 1
+fi
+
+# Prepare new version of schema file.
+# Define custom python path locally, do not let it bleed over to node, since it
+# could interfere with sandbox operation.
+if [[ -e sandbox_venv3/bin/python ]]; then
+  # Use our virtual env if available.
+  PYTHON=sandbox_venv3/bin/python
+else
+  # Fall back on system.
+  PYTHON=python
+fi
+PYTHONPATH=sandbox/grist:sandbox/thirdparty $PYTHON -B sandbox/gen_js_schema.py > $schema_ts.tmp
+
+# Prepare new version of sql file.
+export NODE_PATH=_build:_build/core:_build/stubs:_build/ext
+BUILD=$(test -e _build/core && echo "_build/core" || echo "_build")
+node $BUILD/app/server/generateInitialDocSql.js $sql_ts.tmpdoc > $sql_ts.tmp
+
+rm $sql_ts.tmpdoc.grist
+mv $schema_ts.tmp $schema_ts
+mv $sql_ts.tmp $sql_ts

package.json

@@ -12,11 +12,11 @@
     "install:python3": "buildtools/prepare_python3.sh",
     "build:prod": "buildtools/build.sh",
     "start:prod": "sandbox/run.sh",
-    "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} _build/test/common/*.js _build/test/client/*.js _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js",
+    "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
-    "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 _build/test/nbrowser/**/*.js",
+    "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 '_build/test/nbrowser/**/*.js'",
-    "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/client/**/*.js",
+    "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'",
-    "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/common/**/*.js",
+    "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'",
-    "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/server/**/*.js _build/test/gen-server/**/*.js",
+    "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
     "test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js",
     "test:docker": "./test/test_under_docker.sh",
     "test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}",
@@ -24,7 +24,8 @@
     "lint": "eslint --cache --cache-strategy content .",
     "lint:fix": "eslint  --cache --cache-strategy=content --fix .",
     "lint:ci": "eslint --max-warnings=0 .",
-    "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js"
+    "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js",
+    "generate:schema:ts": "buildtools/update_schema.sh"
   },
   "keywords": [
     "grist",

sandbox/gen_js_schema.py

@@ -32,7 +32,10 @@ import { GristObjCode } from "app/plugin/GristData";
 export const SCHEMA_VERSION = %d;
 
 export const schema = {
-""" % (__file__, schema.SCHEMA_VERSION))
+""" % ('core/sandbox/gen_js_schema.py', schema.SCHEMA_VERSION))
+  # The script name is hardcoded since the Grist sandbox can be
+  # at different paths depending on how Grist is installed, and
+  # we don't want unnecessary changes to generated files.
 
   for table in schema.schema_create_actions():
     print('  "%s": {' % table.table_id)

static/locales/de.client.json

@@ -37,7 +37,9 @@
         "Type a message...": "Geben Sie eine Nachricht ein…",
         "User Attributes": "Benutzer-Attribute",
         "Users": "Benutzer",
-        "View As": "Anzeigen als"
+        "View As": "Anzeigen als",
+        "Seed rules": "Saatgut-Regeln",
+        "When adding table rules, automatically add a rule to grant OWNER full access.": "Beim Hinzufügen von Tabellenregeln wird automatisch eine Regel hinzugefügt, um BESITZER vollen Zugriff zu gewähren."
     },
     "AccountPage": {
         "API": "API",
@@ -824,6 +826,19 @@
         "Example Users": "Beispiel Benutzer"
     },
     "FilterBar": {
+        "SearchColumns": "Spalten suchen",
         "Search Columns": "Spalten suchen"
+    },
+    "ACLUsers": {
+        "Example Users": "Beispiel Benutzer",
+        "Users from table": "Benutzer aus der Tabelle",
+        "View As": "Anzeigen als"
+    },
+    "TypeTransform": {
+        "Apply": "Anwenden",
+        "Cancel": "Abbrechen",
+        "Preview": "Vorschau",
+        "Update formula (Shift+Enter)": "Formel aktualisieren (Umschalttaste+Eingabetaste)",
+        "Revise": "Überarbeiten"
     }
 }

static/locales/en.client.json

@@ -13,7 +13,7 @@
         "Allow everyone to view Access Rules.": "Allow everyone to view Access Rules.",
         "Attribute name": "Attribute name",
         "Attribute to Look Up": "Attribute to Look Up",
-        "Checking...": "Checking...",
+        "Checking...": "Checking…",
         "Condition": "Condition",
         "Default Rules": "Default Rules",
         "Delete Table Rules": "Delete Table Rules",
@@ -34,7 +34,7 @@
         "Save": "Save",
         "Saved": "Saved",
         "Special Rules": "Special Rules",
-        "Type a message...": "Type a message...",
+        "Type a message...": "Type a message…",
         "User Attributes": "User Attributes",
         "View As": "View As",
         "Seed rules": "Seed rules",
@@ -362,7 +362,7 @@
         "Hide {{count}} columns_one": "Hide column",
         "Hide {{count}} columns_other": "Hide {{count}} columns",
         "Insert column to the {{to}}": "Insert column to the {{to}}",
-        "More sort options ...": "More sort options ...",
+        "More sort options ...": "More sort options…",
         "Rename column": "Rename column",
         "Reset {{count}} columns_one": "Reset column",
         "Reset {{count}} columns_other": "Reset {{count}} columns",
@@ -560,7 +560,7 @@
         "Export XLSX": "Export XLSX",
         "Manage Users": "Manage Users",
         "Original": "Original",
-        "Replace {{termToUse}}...": "Replace {{termToUse}}...",
+        "Replace {{termToUse}}...": "Replace {{termToUse}}…",
         "Return to {{termToUse}}": "Return to {{termToUse}}",
         "Save Copy": "Save Copy",
         "Save Document": "Save Document",

static/locales/es.client.json

@@ -32,7 +32,9 @@
         "Remove {{- name }} user attribute": "Eliminar el atributo de usuario {{- name }}",
         "Rules for table ": "Reglas para la tabla ",
         "Special Rules": "Reglas especiales",
-        "View As": "Ver como"
+        "View As": "Ver como",
+        "Seed rules": "Reglas de semillas",
+        "When adding table rules, automatically add a rule to grant OWNER full access.": "Al agregar reglas de tabla, agregue automáticamente una regla para otorgar acceso completo al PROPIETARIO."
     },
     "AccountPage": {
         "API": "API",
@@ -733,6 +735,7 @@
         "Unmark On-Demand": "Desmarcar bajo demanda"
     },
     "FilterBar": {
+        "SearchColumns": "Buscar columnas",
         "Search Columns": "Buscar columnas"
     },
     "Importer": {
@@ -815,5 +818,17 @@
         "Find Previous ": "Buscar anterior ",
         "No results": "Sin resultados",
         "Search in document": "Buscar en el documento"
+    },
+    "ACLUsers": {
+        "Users from table": "Usuarios de la tabla",
+        "View As": "Ver como",
+        "Example Users": "Usuarios de ejemplo"
+    },
+    "TypeTransform": {
+        "Apply": "Aplicar",
+        "Cancel": "Cancelar",
+        "Revise": "Revisar",
+        "Update formula (Shift+Enter)": "Actualizar fórmula (Mayús+Intro)",
+        "Preview": "Vista previa"
     }
 }

static/locales/pt_BR.client.json

@@ -37,7 +37,9 @@
         "Type a message...": "Escreva uma mensagem…",
         "User Attributes": "Atributos de Usuário",
         "Users": "Usuários",
-        "View As": "Ver como"
+        "View As": "Ver como",
+        "Seed rules": "Regras de propagação",
+        "When adding table rules, automatically add a rule to grant OWNER full access.": "Ao adicionar regras de tabela, adicione automaticamente uma regra para conceder ao PROPRIETÁRIO acesso total."
     },
     "AccountPage": {
         "API": "API",
@@ -819,11 +821,24 @@
         "UnknownUser": "Usuário desconhecido"
     },
     "FilterBar": {
+        "SearchColumns": "Procurar colunas",
         "Search Columns": "Procurar colunas"
     },
     "ViewAsDropdown": {
         "View As": "Ver como",
         "Users from table": "Usuários da tabela",
         "Example Users": "Usuários de exemplo"
+    },
+    "ACLUsers": {
+        "Users from table": "Usuários da tabela",
+        "View As": "Ver como",
+        "Example Users": "Usuários de exemplo"
+    },
+    "TypeTransform": {
+        "Apply": "Aplicar",
+        "Cancel": "Cancelar",
+        "Preview": "Pré-visualização",
+        "Revise": "Revisar",
+        "Update formula (Shift+Enter)": "Atualizar a fórmula (Shift+Enter)"
     }
 }

test/server/generateInitialDocSql.ts

@@ -0,0 +1,41 @@
+import { getAppRoot } from 'app/server/lib/places';
+import { createTmpDir } from 'test/server/docTools';
+import * as testUtils from 'test/server/testUtils';
+
+import { assert } from 'chai';
+import * as childProcess from 'child_process';
+import * as fse from 'fs-extra';
+import * as path from 'path';
+import * as util from 'util';
+
+const execFile = util.promisify(childProcess.execFile);
+
+describe('generateInitialDocSql', function() {
+  this.timeout(10000);
+
+  let tmpDir: string;
+
+  testUtils.setTmpLogLevel('fatal');
+
+  before(async function() {
+    tmpDir = await createTmpDir();
+  });
+
+  it('confirms schema and sql files are up to date (run "yarn run generate:schema:ts" on failure)', async function() {
+    let root = getAppRoot();
+    if (await fse.pathExists(path.join(root, 'core'))) {
+      root = path.join(root, 'core');
+    }
+    const newSchemaTs = path.join(tmpDir, 'schema.ts');
+    const newSqlTs = path.join(tmpDir, 'sql.ts');
+    const currentSchemaTs = path.join(root, 'app/common/schema.ts');
+    const currentSqlTs = path.join(root, 'app/server/lib/initialDocSql.ts');
+    await execFile(path.join(getAppRoot(), 'buildtools/update_schema.sh'), [
+      newSchemaTs, newSqlTs,
+    ], { env: process.env });
+    assert.equal((await fse.readFile(newSchemaTs)).toString(),
+                 (await fse.readFile(currentSchemaTs)).toString());
+    assert.equal((await fse.readFile(newSqlTs)).toString(),
+                 (await fse.readFile(currentSqlTs)).toString());
+  });
+});

test/setupPaths.js

@@ -0,0 +1,7 @@
+// enhance require() to support project paths and typescript.
+const path = require('path');
+const appModulePath = require('app-module-path');
+const root = path.dirname(__dirname);
+appModulePath.addPath(path.join(root, "_build"));
+appModulePath.addPath(path.join(root, "_build/core"));
+appModulePath.addPath(path.join(root, "_build/ext"));

test/upgradeDocument

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require('./setupPaths');
+require('test/upgradeDocumentImpl').main().catch(e => console.error(String(e)));

test/upgradeDocumentImpl.ts

@@ -0,0 +1,43 @@
+/**
+ * Upgrade one or more documents (both the DocStorage and schema migrations).
+ *
+ * Usage:
+ *    test/upgradeDocument <docPaths...>
+ */
+import {copyFile} from 'app/server/lib/docUtils';
+import {createDocTools} from 'test/server/docTools';
+import log from 'app/server/lib/log';
+import * as fs from "fs";
+
+export async function main() {
+  const docPaths = process.argv.slice(2);
+  if (docPaths.length === 0) {
+    console.log(`Usage:\n    test/upgradeDocument path/to/doc.grist ...\n`);
+    throw new Error("Document argument required");
+  }
+  for (const docPath of docPaths) {
+    if (!docPath.endsWith('.grist')) {
+      throw new Error(`Document path should have .grist extension: ${docPath}`);
+    }
+    if (!fs.existsSync(docPath)) {
+      throw new Error(`Document path doesn't exist: ${docPath}`);
+    }
+  }
+
+  const prevLogLevel = log.transports.file.level;
+  log.transports.file.level = 'warn';
+  const docTools = createDocTools();
+  await docTools.before();
+  try {
+    for (const docPath of docPaths) {
+      console.log(`Upgrading ${docPath}`);
+      const activeDoc = await docTools.loadLocalDoc(docPath);
+      await activeDoc.waitForInitialization();
+      await activeDoc.shutdown();
+      await copyFile(docTools.getStorageManager().getPath(activeDoc.docName), docPath);
+    }
+  } finally {
+    await docTools.after();
+    log.transports.file.level = prevLogLevel;
+  }
+}