(core) updates from grist-core

.github/workflows/main.yml

@@ -44,6 +44,9 @@ jobs:
       - name: Run smoke test
         run: VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:smoke
 
+      - name: Run python tests
+        run: yarn run test:python
+
       - name: Run main tests
         run: MOCHA_WEBDRIVER_HEADLESS=1 yarn run test
 

README.md

@@ -291,12 +291,42 @@ Testing:
 
 Variable | Purpose
 -------- | -------
-GRIST_TESTING_SOCKET | a socket used for out-of-channel communication during tests only.
+GRIST_TESTING_SOCKET    | a socket used for out-of-channel communication during tests only.
 GRIST_TEST_HTTPS_OFFSET | if set, adds https ports at the specified offset.  This is useful in testing.
-GRIST_TEST_SSL_CERT | if set, contains filename of SSL certificate.
-GRIST_TEST_SSL_KEY  | if set, contains filename of SSL private key.
-GRIST_TEST_LOGIN    | allow fake unauthenticated test logins (suitable for dev environment only).
-GRIST_TEST_ROUTER | if set, then the home server will serve a mock version of router api at /test/router
+GRIST_TEST_SSL_CERT     | if set, contains filename of SSL certificate.
+GRIST_TEST_SSL_KEY      | if set, contains filename of SSL private key.
+GRIST_TEST_LOGIN        | allow fake unauthenticated test logins (suitable for dev environment only).
+GRIST_TEST_ROUTER       | if set, then the home server will serve a mock version of router api at /test/router
+GREP_TESTS              | pattern for selecting specific tests to run (e.g. `env GREP_TESTS=ActionLog yarn test`).
+
+## Tests
+
+Tests are run automatically as part of CI when a PR is opened. However, it can be helpful to run tests locally
+before pushing your changes to GitHub. First, you'll want to make sure you've installed all dependencies:
+
+```
+yarn install
+yarn install:python
+```
+
+Then, you can run the main test suite like so:
+
+```
+yarn test
+```
+
+Python tests may also be run locally. (Note: currently requires Python 3.9.)
+
+```
+yarn test:python
+```
+
+For running specific tests, you can specify a pattern with the `GREP_TESTS` variable:
+
+```
+env GREP_TESTS=ChoiceList yarn test
+env GREP_TESTS=summary yarn test:python
+```
 
 ## License
 

app/client/ui/GridViewMenus.ts

@@ -58,7 +58,7 @@ export function calcFieldsCondition(fields: ViewFieldRec[], condition: (f: ViewF
 }
 
 export function ColumnContextMenu(options: IColumnContextMenu) {
-  const { disableModify, filterOpenFunc, colId, sortSpec, isReadonly, isRaw } = options;
+  const { disableModify, filterOpenFunc, colId, sortSpec, isReadonly } = options;
 
   const disableForReadonlyColumn = dom.cls('disabled', Boolean(disableModify) || isReadonly);
 
@@ -112,7 +112,6 @@ export function ColumnContextMenu(options: IColumnContextMenu) {
     menuItem(allCommands.sortFilterTabOpen.run, 'More sort options ...', testId('more-sort-options')),
     menuDivider({style: 'margin-top: 0;'}),
     menuItemCmd(allCommands.renameField, 'Rename column', disableForReadonlyColumn),
-    menuItemCmd(allCommands.hideFields, 'Hide column', dom.cls('disabled', isReadonly || isRaw)),
     freezeMenuItemCmd(options),
     menuDivider(),
     MultiColumnMenu((options.disableFrozenMenu = true, options)),
@@ -149,7 +148,7 @@ export function MultiColumnMenu(options: IMultiColumnContextMenu) {
     (options.isFormula !== true ?
       menuItemCmd(allCommands.clearValues, 'Clear values', disableForReadonlyColumn) : null),
 
-    menuItemCmd(allCommands.hideFields, nameHideColumns, disableForReadonlyView),
+    (!options.isRaw ? menuItemCmd(allCommands.hideFields, nameHideColumns, disableForReadonlyView) : null),
     menuItemCmd(allCommands.clearColumns, nameClearColumns, disableForReadonlyColumn),
     menuItemCmd(allCommands.deleteFields, nameDeleteColumns, disableForReadonlyColumn),
 

app/client/ui/HomeIntro.ts

@@ -110,7 +110,7 @@ function makePersonalIntro(homeModel: HomeModel, user: FullUser) {
 }
 
 function makeAnonIntro(homeModel: HomeModel) {
-  const signUp = cssLink({href: getLoginOrSignupUrl()}, 'Sign up');
+  const signUp = cssLink({href: getLoginOrSignupUrl()}, t('SignUp'));
   return [
     css.docListHeader(t('Welcome'), testId('welcome-title')),
     cssIntroLine('Get started by exploring templates, or creating your first Grist document.'),

documentation/translations.md

@@ -0,0 +1,150 @@
+# Internationalization and Localization
+
+## General description
+
+Localization support (translations) in Grist is implemented via
+[https://www.i18next.com](https://www.i18next.com/overview/plugins-and-utils) javascript library. It
+is used both on the server (node) and the client side (browser). It has very good documentation,
+supports all needed features (like interpolation, pluralization and context), and has a rich plugin
+ecosystem. It is also very popular and widely used.
+
+## Localization setup
+
+Resource files are located in a `static/locales` directory, but Grist can be configured to read them
+from any other location by using the `GRIST_LOCALES_DIR` environmental variable. All resource files
+are read when the server starts. The default and required language code is `en` (English), all other
+languages are optional and will be supported if server can find a resource file with proper language
+code. Languages are resolved hierarchically, from most specific to a general one, for example, for
+Polish code _pl-PL_, the library will first try _pl-PL_, then _pl_, and then will fallback to a
+default language _en_ (https://www.i18next.com/principles/translation-resolution).
+
+All language variants (e.g., _fr-FR_, _pl-PL_, _en-UK_) are supported if Grist can find a main
+language resource file. For example, to support a _fr-FR_ language code, Grist expects to have at
+least _fr.core.json_ file. The main language file will be used as a default fallback for all French
+language codes like _fr-FR_ or _fr-CA_, in case there is no resource file for a specif variant (like
+`fr-CA.core.json`) or some keys are missing from the variant file.
+
+Here is an example of a language resource file `en.core.json` currently used by Grist:
+
+```json
+{
+  "Welcome": "Welcome to Grist!",
+  "Loading": "Loading",
+  "AddNew": "Add New",
+  "OtherSites": "Other Sites",
+  "OtherSitesWelcome": "Your are on {{siteName}}. You also have access to the following sites:",
+  "OtherSitesWelcome_personal": "Your are on your personal site. You also have access to the following sites:",
+  "AllDocuments": "All Documents",
+  "ExamplesAndTemplates": "Examples and Templates",
+  "MoreExamplesAndTemplates": "More Examples and Templates"
+}
+```
+
+It maps a key to a translated message. It also has an example of interpolation and context features
+in the `OtherSitesWelcome` resource key. More information about how to use those features can be
+found at https://www.i18next.com/translation-function/interpolation and
+https://www.i18next.com/translation-function/context.
+
+Both client and server code (node.js) use the same resource files. A resource file name format
+follows a pattern: [language code].[product].json (i.e. `pl-Pl.core.json`, `en-US.core.json`,
+`en.core.json`). Grist can be packaged as several different products, and each product can have its
+own translation files that are added to the core. Products are supported by leveraging `i18next`
+feature called `namespaces` https://www.i18next.com/principles/namespaces.
+
+## Translation instruction
+
+### Client
+
+The entry point for all translations is a function exported from 'app/client/lib/localization'.
+
+```ts
+import { t } from 'app/client/lib/localization';
+```
+
+It is a wrapper around `i18next` exported method with the same interface
+https://www.i18next.com/overview/api#t. As a future improvement, all resource keys used in
+translation files will be extracted and converted to a TypeScript definition file, for a “compile”
+time error detection and and better development experience. Here are couple examples how this method
+is used:
+
+_app/client/ui.DocMenu.ts_
+
+```ts
+  css.otherSitesHeader(
+    t('OtherSites'),
+    .....
+  ),
+  dom.maybe((use) => !use(hideOtherSitesObs), () => {
+    const personal = Boolean(home.app.currentOrg?.owner);
+    const siteName = home.app.currentOrgName;
+    return [
+      dom('div',
+        t('OtherSitesWelcome', { siteName, context: personal ? 'personal' : '' }),
+        testId('other-sites-message')
+```
+
+_app/client/ui/HomeIntro.ts_
+
+```ts
+function makeAnonIntro(homeModel: HomeModel) {
+  const signUp = cssLink({href: getLoginOrSignupUrl()}, t('SignUp'));
+  return [
+    css.docListHeader(t('Welcome'), testId('welcome-title')),
+```
+
+Some things are not supported at this moment and will need to be addressed in future development
+tasks:
+
+- Date time picker component. It has its own resource files that are already imported by Grist but
+  not used in the main application. https://bootstrap-datepicker.readthedocs.io/en/latest/i18n.html
+- Static HTML files used as a placeholder (for example, for Custom widgets).
+- Formatting dates. Grist is using `moment.js` library, which has its own i18n support. Date formats
+  used by Grist are shared between client, server and sandbox code and are not compatible with
+  `i18next` library.
+
+### Server
+
+For server-side code, Grist is using https://github.com/i18next/i18next-http-middleware plugin,
+which exposes `i18next` API in the `Request` object. It automatically detects user language (from
+request headers) and configures all API methods to use the proper language (either requested by the
+client or a default one). `Comm` object and `webSocket` API use a very similar approach, each
+`Client` object has its own instance of `i18next` library configured with a proper language (also
+detected from the HTTP headers).
+
+Naturally, most of the text that should be translated on the server side is used by the Error
+handlers. This requires a significant amount of work to change how errors are reported to the
+client, and it is still in a design state.
+
+Here is an example of how to use the API to translate a message from an HTTP endpoint in
+`HomeServer`.
+
+_app/server/lib/sendAppPage.ts_
+
+```ts
+function getPageTitle(req: express.Request, config: GristLoadConfig): string {
+  const maybeDoc = getDocFromConfig(config);
+  if (!maybeDoc) {
+    return req.t('Loading') + '...';
+  }
+
+  return handlebars.Utils.escapeExpression(maybeDoc.name);
+}
+```
+
+### Next steps
+
+- Annotate all client code and create all resource files in `en.core.json` file. Almost all static
+  text is ready for translation.
+- Store language settings with the user profile and allow a user to change it on the Account Page.
+  Consider also adding a cookie-based solution that custom widgets can use, or extend the
+  **WidgetFrame** component so that it can pass current user language to the hosted widget page.
+- Generate type declaration files at build time to provide `missing key` error detection as soon as
+  possible.
+- Dynamically Include calendar control language resource files based on the currently selected
+  language.
+- Refactor server-side code that is handling errors or creating user-facing messages. Currently,
+  error messages are created at the place where the Error has occurred. Preferably errors should
+  include error codes and all information needed to assemble the error message by the client code.
+- Add localization support to the `moment.js` library to format dates properly according to the
+  currently selected language.
+- Add support for custom HTML page translation. For example `custom-widget.html`

package.json

@@ -13,11 +13,13 @@
     "build:prod": "buildtools/build.sh",
     "start:prod": "sandbox/run.sh",
     "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha -g ${GREP_TESTS:-''} _build/test/common/*.js _build/test/client/*.js _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js",
+    "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha -g ${GREP_TESTS:-''} --slow 6000 _build/test/nbrowser/**/*.js",
     "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/client/**/*.js",
     "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/common/**/*.js",
     "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/server/**/*.js _build/test/gen-server/**/*.js",
     "test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js",
     "test:docker": "./test/test_under_docker.sh",
+    "test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}",
     "cli": "NODE_PATH=_build:_build/stubs:_build/ext node _build/app/server/companion.js"
   },
   "keywords": [

static/locales/en.core.json

@@ -1,5 +1,6 @@
 {
   "Welcome": "Welcome to Grist!",
+  "SignUp": "Sign up",
   "Loading": "Loading",
   "AddNew": "Add New",
   "OtherSites": "Other Sites",

test/deployment/ActionLog.ts

@@ -0,0 +1 @@
+import "test/nbrowser/ActionLog";

test/deployment/ChoiceList.ts

@@ -0,0 +1 @@
+import "test/nbrowser/ChoiceList";

test/deployment/DuplicateDocument.ts

@@ -0,0 +1 @@
+import "test/nbrowser/DuplicateDocument";

test/deployment/Fork.ts

@@ -0,0 +1 @@
+import "test/nbrowser/Fork";

test/deployment/HomeIntro.ts

@@ -0,0 +1 @@
+import "test/nbrowser/HomeIntro";

test/deployment/Pages.ts

@@ -0,0 +1 @@
+import "test/nbrowser/Pages";

test/deployment/README.md

@@ -0,0 +1,4 @@
+# Deployment tests
+
+Link or import here all tests that can be run against an external server or
+a docker container (i.e: tests that don't rely on in-memory TestServer).

test/deployment/ReferenceColumns.ts

@@ -0,0 +1 @@
+import "test/nbrowser/ReferenceColumns";

test/deployment/ReferenceList.ts

@@ -0,0 +1 @@
+import "test/nbrowser/ReferenceList";

test/deployment/Smoke.ts

@@ -0,0 +1 @@
+import "test/nbrowser/Smoke";

test/nbrowser/Localization.ts

@@ -117,7 +117,10 @@ describe("Localization", function() {
     });
   });
 
-  it("breaks the server if something is wrong with resource files", async () => {
+  it("breaks the server if something is wrong with resource files", async function() {
+    if (server.isExternalServer()) {
+      this.skip();
+    }
     const oldEnv = new testUtils.EnvironmentSnapshot();
     try {
       // Wrong path to locales.

test/test_under_docker.sh

@@ -64,4 +64,4 @@ TEST_ADD_SAMPLES=1 TEST_ACCOUNT_PASSWORD=not-needed \
   GRIST_SESSION_COOKIE=grist_test_cookie \
   GRIST_TEST_LOGIN=1 \
   NODE_PATH=_build:_build/stubs \
-  $MOCHA _build/test/nbrowser/*.js -g ${GREP_TESTS:-''} "$@"
+  $MOCHA _build/test/deployment/*.js --slow 6000 -g ${GREP_TESTS:-''} "$@"