(core) updates from grist-core

messytables-requirements
Paul Fitzpatrick 2 years ago
commit 303d071de1

@ -44,6 +44,9 @@ jobs:
- name: Run smoke test
run: VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:smoke
- name: Run python tests
run: yarn run test:python
- name: Run main tests
run: MOCHA_WEBDRIVER_HEADLESS=1 yarn run test

@ -291,12 +291,42 @@ Testing:
Variable | Purpose
-------- | -------
GRIST_TESTING_SOCKET | a socket used for out-of-channel communication during tests only.
GRIST_TESTING_SOCKET | a socket used for out-of-channel communication during tests only.
GRIST_TEST_HTTPS_OFFSET | if set, adds https ports at the specified offset. This is useful in testing.
GRIST_TEST_SSL_CERT | if set, contains filename of SSL certificate.
GRIST_TEST_SSL_KEY | if set, contains filename of SSL private key.
GRIST_TEST_LOGIN | allow fake unauthenticated test logins (suitable for dev environment only).
GRIST_TEST_ROUTER | if set, then the home server will serve a mock version of router api at /test/router
GRIST_TEST_SSL_CERT | if set, contains filename of SSL certificate.
GRIST_TEST_SSL_KEY | if set, contains filename of SSL private key.
GRIST_TEST_LOGIN | allow fake unauthenticated test logins (suitable for dev environment only).
GRIST_TEST_ROUTER | if set, then the home server will serve a mock version of router api at /test/router
GREP_TESTS | pattern for selecting specific tests to run (e.g. `env GREP_TESTS=ActionLog yarn test`).
## Tests
Tests are run automatically as part of CI when a PR is opened. However, it can be helpful to run tests locally
before pushing your changes to GitHub. First, you'll want to make sure you've installed all dependencies:
```
yarn install
yarn install:python
```
Then, you can run the main test suite like so:
```
yarn test
```
Python tests may also be run locally. (Note: currently requires Python 3.9.)
```
yarn test:python
```
For running specific tests, you can specify a pattern with the `GREP_TESTS` variable:
```
env GREP_TESTS=ChoiceList yarn test
env GREP_TESTS=summary yarn test:python
```
## License

@ -58,7 +58,7 @@ export function calcFieldsCondition(fields: ViewFieldRec[], condition: (f: ViewF
}
export function ColumnContextMenu(options: IColumnContextMenu) {
const { disableModify, filterOpenFunc, colId, sortSpec, isReadonly, isRaw } = options;
const { disableModify, filterOpenFunc, colId, sortSpec, isReadonly } = options;
const disableForReadonlyColumn = dom.cls('disabled', Boolean(disableModify) || isReadonly);
@ -112,7 +112,6 @@ export function ColumnContextMenu(options: IColumnContextMenu) {
menuItem(allCommands.sortFilterTabOpen.run, 'More sort options ...', testId('more-sort-options')),
menuDivider({style: 'margin-top: 0;'}),
menuItemCmd(allCommands.renameField, 'Rename column', disableForReadonlyColumn),
menuItemCmd(allCommands.hideFields, 'Hide column', dom.cls('disabled', isReadonly || isRaw)),
freezeMenuItemCmd(options),
menuDivider(),
MultiColumnMenu((options.disableFrozenMenu = true, options)),
@ -149,7 +148,7 @@ export function MultiColumnMenu(options: IMultiColumnContextMenu) {
(options.isFormula !== true ?
menuItemCmd(allCommands.clearValues, 'Clear values', disableForReadonlyColumn) : null),
menuItemCmd(allCommands.hideFields, nameHideColumns, disableForReadonlyView),
(!options.isRaw ? menuItemCmd(allCommands.hideFields, nameHideColumns, disableForReadonlyView) : null),
menuItemCmd(allCommands.clearColumns, nameClearColumns, disableForReadonlyColumn),
menuItemCmd(allCommands.deleteFields, nameDeleteColumns, disableForReadonlyColumn),

@ -110,7 +110,7 @@ function makePersonalIntro(homeModel: HomeModel, user: FullUser) {
}
function makeAnonIntro(homeModel: HomeModel) {
const signUp = cssLink({href: getLoginOrSignupUrl()}, 'Sign up');
const signUp = cssLink({href: getLoginOrSignupUrl()}, t('SignUp'));
return [
css.docListHeader(t('Welcome'), testId('welcome-title')),
cssIntroLine('Get started by exploring templates, or creating your first Grist document.'),

@ -0,0 +1,150 @@
# Internationalization and Localization
## General description
Localization support (translations) in Grist is implemented via
[https://www.i18next.com](https://www.i18next.com/overview/plugins-and-utils) javascript library. It
is used both on the server (node) and the client side (browser). It has very good documentation,
supports all needed features (like interpolation, pluralization and context), and has a rich plugin
ecosystem. It is also very popular and widely used.
## Localization setup
Resource files are located in a `static/locales` directory, but Grist can be configured to read them
from any other location by using the `GRIST_LOCALES_DIR` environmental variable. All resource files
are read when the server starts. The default and required language code is `en` (English), all other
languages are optional and will be supported if server can find a resource file with proper language
code. Languages are resolved hierarchically, from most specific to a general one, for example, for
Polish code _pl-PL_, the library will first try _pl-PL_, then _pl_, and then will fallback to a
default language _en_ (https://www.i18next.com/principles/translation-resolution).
All language variants (e.g., _fr-FR_, _pl-PL_, _en-UK_) are supported if Grist can find a main
language resource file. For example, to support a _fr-FR_ language code, Grist expects to have at
least _fr.core.json_ file. The main language file will be used as a default fallback for all French
language codes like _fr-FR_ or _fr-CA_, in case there is no resource file for a specif variant (like
`fr-CA.core.json`) or some keys are missing from the variant file.
Here is an example of a language resource file `en.core.json` currently used by Grist:
```json
{
"Welcome": "Welcome to Grist!",
"Loading": "Loading",
"AddNew": "Add New",
"OtherSites": "Other Sites",
"OtherSitesWelcome": "Your are on {{siteName}}. You also have access to the following sites:",
"OtherSitesWelcome_personal": "Your are on your personal site. You also have access to the following sites:",
"AllDocuments": "All Documents",
"ExamplesAndTemplates": "Examples and Templates",
"MoreExamplesAndTemplates": "More Examples and Templates"
}
```
It maps a key to a translated message. It also has an example of interpolation and context features
in the `OtherSitesWelcome` resource key. More information about how to use those features can be
found at https://www.i18next.com/translation-function/interpolation and
https://www.i18next.com/translation-function/context.
Both client and server code (node.js) use the same resource files. A resource file name format
follows a pattern: [language code].[product].json (i.e. `pl-Pl.core.json`, `en-US.core.json`,
`en.core.json`). Grist can be packaged as several different products, and each product can have its
own translation files that are added to the core. Products are supported by leveraging `i18next`
feature called `namespaces` https://www.i18next.com/principles/namespaces.
## Translation instruction
### Client
The entry point for all translations is a function exported from 'app/client/lib/localization'.
```ts
import { t } from 'app/client/lib/localization';
```
It is a wrapper around `i18next` exported method with the same interface
https://www.i18next.com/overview/api#t. As a future improvement, all resource keys used in
translation files will be extracted and converted to a TypeScript definition file, for a “compile”
time error detection and and better development experience. Here are couple examples how this method
is used:
_app/client/ui.DocMenu.ts_
```ts
css.otherSitesHeader(
t('OtherSites'),
.....
),
dom.maybe((use) => !use(hideOtherSitesObs), () => {
const personal = Boolean(home.app.currentOrg?.owner);
const siteName = home.app.currentOrgName;
return [
dom('div',
t('OtherSitesWelcome', { siteName, context: personal ? 'personal' : '' }),
testId('other-sites-message')
```
_app/client/ui/HomeIntro.ts_
```ts
function makeAnonIntro(homeModel: HomeModel) {
const signUp = cssLink({href: getLoginOrSignupUrl()}, t('SignUp'));
return [
css.docListHeader(t('Welcome'), testId('welcome-title')),
```
Some things are not supported at this moment and will need to be addressed in future development
tasks:
- Date time picker component. It has its own resource files that are already imported by Grist but
not used in the main application. https://bootstrap-datepicker.readthedocs.io/en/latest/i18n.html
- Static HTML files used as a placeholder (for example, for Custom widgets).
- Formatting dates. Grist is using `moment.js` library, which has its own i18n support. Date formats
used by Grist are shared between client, server and sandbox code and are not compatible with
`i18next` library.
### Server
For server-side code, Grist is using https://github.com/i18next/i18next-http-middleware plugin,
which exposes `i18next` API in the `Request` object. It automatically detects user language (from
request headers) and configures all API methods to use the proper language (either requested by the
client or a default one). `Comm` object and `webSocket` API use a very similar approach, each
`Client` object has its own instance of `i18next` library configured with a proper language (also
detected from the HTTP headers).
Naturally, most of the text that should be translated on the server side is used by the Error
handlers. This requires a significant amount of work to change how errors are reported to the
client, and it is still in a design state.
Here is an example of how to use the API to translate a message from an HTTP endpoint in
`HomeServer`.
_app/server/lib/sendAppPage.ts_
```ts
function getPageTitle(req: express.Request, config: GristLoadConfig): string {
const maybeDoc = getDocFromConfig(config);
if (!maybeDoc) {
return req.t('Loading') + '...';
}
return handlebars.Utils.escapeExpression(maybeDoc.name);
}
```
### Next steps
- Annotate all client code and create all resource files in `en.core.json` file. Almost all static
text is ready for translation.
- Store language settings with the user profile and allow a user to change it on the Account Page.
Consider also adding a cookie-based solution that custom widgets can use, or extend the
**WidgetFrame** component so that it can pass current user language to the hosted widget page.
- Generate type declaration files at build time to provide `missing key` error detection as soon as
possible.
- Dynamically Include calendar control language resource files based on the currently selected
language.
- Refactor server-side code that is handling errors or creating user-facing messages. Currently,
error messages are created at the place where the Error has occurred. Preferably errors should
include error codes and all information needed to assemble the error message by the client code.
- Add localization support to the `moment.js` library to format dates properly according to the
currently selected language.
- Add support for custom HTML page translation. For example `custom-widget.html`

@ -13,11 +13,13 @@
"build:prod": "buildtools/build.sh",
"start:prod": "sandbox/run.sh",
"test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha -g ${GREP_TESTS:-''} _build/test/common/*.js _build/test/client/*.js _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js",
"test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha -g ${GREP_TESTS:-''} --slow 6000 _build/test/nbrowser/**/*.js",
"test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/client/**/*.js",
"test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/common/**/*.js",
"test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/server/**/*.js _build/test/gen-server/**/*.js",
"test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js",
"test:docker": "./test/test_under_docker.sh",
"test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}",
"cli": "NODE_PATH=_build:_build/stubs:_build/ext node _build/app/server/companion.js"
},
"keywords": [

@ -1,5 +1,6 @@
{
"Welcome": "Welcome to Grist!",
"SignUp": "Sign up",
"Loading": "Loading",
"AddNew": "Add New",
"OtherSites": "Other Sites",

@ -0,0 +1 @@
import "test/nbrowser/ActionLog";

@ -0,0 +1 @@
import "test/nbrowser/ChoiceList";

@ -0,0 +1 @@
import "test/nbrowser/DuplicateDocument";

@ -0,0 +1 @@
import "test/nbrowser/Fork";

@ -0,0 +1 @@
import "test/nbrowser/HomeIntro";

@ -0,0 +1 @@
import "test/nbrowser/Pages";

@ -0,0 +1,4 @@
# Deployment tests
Link or import here all tests that can be run against an external server or
a docker container (i.e: tests that don't rely on in-memory TestServer).

@ -0,0 +1 @@
import "test/nbrowser/ReferenceColumns";

@ -0,0 +1 @@
import "test/nbrowser/ReferenceList";

@ -0,0 +1 @@
import "test/nbrowser/Smoke";

@ -117,7 +117,10 @@ describe("Localization", function() {
});
});
it("breaks the server if something is wrong with resource files", async () => {
it("breaks the server if something is wrong with resource files", async function() {
if (server.isExternalServer()) {
this.skip();
}
const oldEnv = new testUtils.EnvironmentSnapshot();
try {
// Wrong path to locales.

@ -64,4 +64,4 @@ TEST_ADD_SAMPLES=1 TEST_ACCOUNT_PASSWORD=not-needed \
GRIST_SESSION_COOKIE=grist_test_cookie \
GRIST_TEST_LOGIN=1 \
NODE_PATH=_build:_build/stubs \
$MOCHA _build/test/nbrowser/*.js -g ${GREP_TESTS:-''} "$@"
$MOCHA _build/test/deployment/*.js --slow 6000 -g ${GREP_TESTS:-''} "$@"

Loading…
Cancel
Save