From 5219932a1f94d99f48f708df173e79f219a1ee0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?= Date: Thu, 29 Sep 2022 10:01:37 +0200 Subject: [PATCH] (core) i18 Summary: Adding initial work for localization support. Summary in https://grist.quip.com/OtZKA6RHdQ6T/Internationalization-and-Localization Test Plan: Not yet Reviewers: paulfitz Reviewed By: paulfitz Subscribers: paulfitz Differential Revision: https://phab.getgrist.com/D3633 --- app/client/app.js | 12 ++- app/client/lib/localization.ts | 86 +++++++++++++++++ app/client/models/HomeModel.ts | 7 +- app/client/ui/AddNewButton.ts | 3 +- app/client/ui/DocMenu.ts | 13 +-- app/client/ui/HomeIntro.ts | 3 +- app/client/ui/setupPage.ts | 3 + app/common/gristUrls.ts | 6 ++ app/server/lib/Client.ts | 10 ++ app/server/lib/Comm.ts | 4 +- app/server/lib/FlexServer.ts | 24 ++++- app/server/lib/sendAppPage.ts | 16 +++- app/server/localization.ts | 75 +++++++++++++++ package.json | 4 + static/account.html | 1 + static/activation.html | 1 + static/app.html | 1 + static/error.html | 1 + static/locales/en.core.json | 11 +++ test/nbrowser/Localization.ts | 169 +++++++++++++++++++++++++++++++++ yarn.lock | 36 +++++++ 21 files changed, 471 insertions(+), 15 deletions(-) create mode 100644 app/client/lib/localization.ts create mode 100644 app/server/localization.ts create mode 100644 static/locales/en.core.json create mode 100644 test/nbrowser/Localization.ts diff --git a/app/client/app.js b/app/client/app.js index 15ca0baa..996e6138 100644 --- a/app/client/app.js +++ b/app/client/app.js @@ -8,6 +8,8 @@ if (window._gristAppLoaded) { } window._gristAppLoaded = true; +const {setupLocale} = require('./lib/localization'); + const {App} = require('./ui/App'); // Disable longStackTraces, which seem to be enabled in the browser by default. @@ -28,7 +30,14 @@ $(function() { if (event.persisted) { window.location.reload(); } }; - window.gristApp = App.create(null); + const localeSetup = setupLocale(); + // By the time dom ready is fired, resource files should already be loaded, but + // if that is not the case, we will redirect to an error page by throwing an error. + localeSetup.then(() => { + window.gristApp = App.create(null); + }).catch(error => { + throw new Error(`Failed to load locale: ${error?.message || 'Unknown error'}`); + }) // Set from the login tests to stub and un-stub functions during execution. window.loginTestSandbox = null; @@ -46,4 +55,5 @@ $(function() { .then(() => window.exposedModules._loadScript(name)); } }; + }); diff --git a/app/client/lib/localization.ts b/app/client/lib/localization.ts new file mode 100644 index 00000000..cae955a5 --- /dev/null +++ b/app/client/lib/localization.ts @@ -0,0 +1,86 @@ +import {getGristConfig} from 'app/common/urlUtils'; +import i18next from 'i18next'; + +export async function setupLocale() { + const now = Date.now(); + const supportedLngs = getGristConfig().supportedLngs ?? ['en']; + let lng = window.navigator.language || 'en'; + // If user agent language is not in the list of supported languages, use the default one. + if (!supportedLngs.includes(lng)) { + // Test if server supports general language. + if (lng.includes("-") && supportedLngs.includes(lng.split("-")[0])) { + lng = lng.split("-")[0]!; + } else { + lng = 'en'; + } + } + + const ns = getGristConfig().namespaces ?? ['core']; + // Initialize localization plugin + try { + // We don't await this promise, as it is resolved synchronously due to initImmediate: false. + i18next.init({ + // By default we use english language. + fallbackLng: 'en', + // Fallback from en-US, en-GB, etc to en. + nonExplicitSupportedLngs: true, + // We will load resources ourselves. + initImmediate: false, + // Read language from navigator object. + lng, + // By default we use core namespace. + defaultNS: 'core', + // Read namespaces that are supported by the server. + // TODO: this can be converted to a dynamic list of namespaces, for async components. + // for now just import all what server offers. + // We can fallback to core namespace for any addons. + fallbackNS: 'core', + ns, + supportedLngs + }).catch((err: any) => { + // This should not happen, the promise should be resolved synchronously, without + // any errors reported. + console.error("i18next failed unexpectedly", err); + }); + // Detect what is resolved languages to load. + const languages = i18next.languages; + // Fetch all json files (all of which should be already preloaded); + const loadPath = `${document.baseURI}locales/{{lng}}.{{ns}}.json`; + const pathsToLoad: Promise[] = []; + async function load(lang: string, n: string) { + const resourceUrl = loadPath.replace('{{lng}}', lang).replace('{{ns}}', n); + const response = await fetch(resourceUrl); + if (!response.ok) { + throw new Error(`Failed to load ${resourceUrl}`); + } + i18next.addResourceBundle(lang, n, await response.json()); + } + for (const lang of languages) { + for (const n of ns) { + pathsToLoad.push(load(lang, n)); + } + } + await Promise.all(pathsToLoad); + console.log("Localization initialized in " + (Date.now() - now) + "ms"); + } catch (error: any) { + reportError(error); + } +} + +/** + * Resolves the translation of the given key, using the given options. + */ +export function t(key: string, args?: any): string { + if (!i18next.exists(key)) { + const error = new Error(`Missing translation for key: ${key} and language: ${i18next.language}`); + reportError(error); + } + return i18next.t(key, args); +} + +/** + * Checks if the given key exists in the any supported language. + */ +export function hasTranslation(key: string) { + return i18next.exists(key); +} diff --git a/app/client/models/HomeModel.ts b/app/client/models/HomeModel.ts index 808e010d..01da3e92 100644 --- a/app/client/models/HomeModel.ts +++ b/app/client/models/HomeModel.ts @@ -264,6 +264,9 @@ export class HomeModelImpl extends Disposable implements HomeModel, ViewSettings // Fetches and updates workspaces, which include contained docs as well. private async _updateWorkspaces() { + if (this.isDisposed()) { + return; + } const org = this._app.currentOrg; if (!org) { this.workspaces.set([]); @@ -285,7 +288,9 @@ export class HomeModelImpl extends Disposable implements HomeModel, ViewSettings this.loading.set("slow"); } const [wss, trashWss, templateWss] = await promise; - + if (this.isDisposed()) { + return; + } // bundleChanges defers computeds' evaluations until all changes have been applied. bundleChanges(() => { this.workspaces.set(wss || []); diff --git a/app/client/ui/AddNewButton.ts b/app/client/ui/AddNewButton.ts index 4442e5c8..6a7a6ede 100644 --- a/app/client/ui/AddNewButton.ts +++ b/app/client/ui/AddNewButton.ts @@ -1,13 +1,14 @@ import {theme, vars} from 'app/client/ui2018/cssVars'; import {icon} from 'app/client/ui2018/icons'; import {dom, DomElementArg, Observable, styled} from "grainjs"; +import {t} from 'app/client/lib/localization'; export function addNewButton(isOpen: Observable | boolean = true, ...args: DomElementArg[]) { return cssAddNewButton( cssAddNewButton.cls('-open', isOpen), // Setting spacing as flex items allows them to shrink faster when there isn't enough space. cssLeftMargin(), - cssAddText('Add New'), + cssAddText(t('AddNew')), dom('div', {style: 'flex: 1 1 16px'}), cssPlusButton(cssPlusIcon('Plus')), dom('div', {style: 'flex: 0 1 16px'}), diff --git a/app/client/ui/DocMenu.ts b/app/client/ui/DocMenu.ts index d4aa094b..c5dc1177 100644 --- a/app/client/ui/DocMenu.ts +++ b/app/client/ui/DocMenu.ts @@ -29,6 +29,7 @@ import {Document, Workspace} from 'app/common/UserAPI'; import {computed, Computed, dom, DomArg, DomContents, IDisposableOwner, makeTestId, observable, Observable} from 'grainjs'; import {buildTemplateDocs} from 'app/client/ui/TemplateDocs'; +import {t} from 'app/client/lib/localization'; import {localStorageBoolObs} from 'app/client/lib/localStorageObs'; import {bigBasicButton} from 'app/client/ui2018/buttons'; import sortBy = require('lodash/sortBy'); @@ -104,10 +105,10 @@ function createLoadedDocMenu(owner: IDisposableOwner, home: HomeModel) { null : css.docListHeader( ( - page === 'all' ? 'All Documents' : + page === 'all' ? t('AllDocuments') : page === 'templates' ? dom.domComputed(use => use(home.featuredTemplates).length > 0, (hasFeaturedTemplates) => - hasFeaturedTemplates ? 'More Examples & Templates' : 'Examples & Templates' + hasFeaturedTemplates ? t('MoreExamplesAndTemplates') : t('ExamplesAndTemplates') ) : page === 'trash' ? 'Trash' : workspace && [css.docHeaderIcon('Folder'), workspaceName(home.app, workspace)] @@ -267,7 +268,7 @@ function buildOtherSites(home: HomeModel) { return css.otherSitesBlock( dom.autoDispose(hideOtherSitesObs), css.otherSitesHeader( - 'Other Sites', + t('OtherSites'), dom.domComputed(hideOtherSitesObs, (collapsed) => collapsed ? css.otherSitesHeaderIcon('Expand') : css.otherSitesHeaderIcon('Collapse') ), @@ -275,11 +276,11 @@ function buildOtherSites(home: HomeModel) { testId('other-sites-header'), ), dom.maybe((use) => !use(hideOtherSitesObs), () => { - const onPersonalSite = Boolean(home.app.currentOrg?.owner); - const siteName = onPersonalSite ? 'your personal site' : `the ${home.app.currentOrgName} site`; + const personal = Boolean(home.app.currentOrg?.owner); + const siteName = home.app.currentOrgName; return [ dom('div', - `You are on ${siteName}. You also have access to the following sites:`, + t('OtherSitesWelcome', { siteName, context: personal ? 'personal' : '' }), testId('other-sites-message') ), css.otherSitesButtons( diff --git a/app/client/ui/HomeIntro.ts b/app/client/ui/HomeIntro.ts index a7ed60dd..8ed16ed1 100644 --- a/app/client/ui/HomeIntro.ts +++ b/app/client/ui/HomeIntro.ts @@ -1,3 +1,4 @@ +import {t} from 'app/client/lib/localization'; import {getLoginOrSignupUrl, urlState} from 'app/client/models/gristUrlState'; import {HomeModel} from 'app/client/models/HomeModel'; import {productPill} from 'app/client/ui/AppHeader'; @@ -111,7 +112,7 @@ function makePersonalIntro(homeModel: HomeModel, user: FullUser) { function makeAnonIntro(homeModel: HomeModel) { const signUp = cssLink({href: getLoginOrSignupUrl()}, 'Sign up'); return [ - css.docListHeader(`Welcome to Grist!`, testId('welcome-title')), + css.docListHeader(t('Welcome'), testId('welcome-title')), cssIntroLine('Get started by exploring templates, or creating your first Grist document.'), cssIntroLine(signUp, ' to save your work.', (shouldHideUiElement('helpCenter') ? null : [' Visit our ', helpCenterLink(), ' to learn more.']), diff --git a/app/client/ui/setupPage.ts b/app/client/ui/setupPage.ts index 6d2c9dbb..dd0307d6 100644 --- a/app/client/ui/setupPage.ts +++ b/app/client/ui/setupPage.ts @@ -1,4 +1,5 @@ import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals'; +import {setupLocale} from 'app/client/lib/localization'; import {AppModel, TopAppModelImpl} from 'app/client/models/AppModel'; import {setUpErrorHandling} from 'app/client/models/errors'; import {buildSnackbarDom} from 'app/client/ui/NotifyUI'; @@ -19,6 +20,8 @@ export function setupPage(buildPage: (appModel: AppModel) => DomContents) { attachCssRootVars(topAppModel.productFlavor); addViewportTag(); + void setupLocale(); + // Add globals needed by test utils. G.window.gristApp = { testNumPendingApiRequests: () => BaseAPI.numPendingRequests(), diff --git a/app/common/gristUrls.ts b/app/common/gristUrls.ts index b4b4746d..5c911931 100644 --- a/app/common/gristUrls.ts +++ b/app/common/gristUrls.ts @@ -562,6 +562,12 @@ export interface GristLoadConfig { // If custom CSS should be included in the head of each page. enableCustomCss?: boolean; + + // Supported languages for the UI. By default only english (en) is supported. + supportedLngs?: readonly string[]; + + // Loaded namespaces for translations. + namespaces?: readonly string[]; } export const HideableUiElements = StringUnion("helpCenter", "billing", "templates", "multiSite", "multiAccounts"); diff --git a/app/server/lib/Client.ts b/app/server/lib/Client.ts index c2221a69..b3b87592 100644 --- a/app/server/lib/Client.ts +++ b/app/server/lib/Client.ts @@ -16,6 +16,7 @@ import log from 'app/server/lib/log'; import {LogMethods} from "app/server/lib/LogMethods"; import {shortDesc} from 'app/server/lib/shortDesc'; import {fromCallback} from 'app/server/lib/serverUtils'; +import {i18n} from 'i18next'; import * as crypto from 'crypto'; import moment from 'moment'; import * as WebSocket from 'ws'; @@ -92,17 +93,26 @@ export class Client { // Identifier for the current GristWSConnection object connected to this client. private _counter: string|null = null; + private _i18Instance?: i18n; constructor( private _comm: Comm, private _methods: Map, private _locale: string, + i18Instance?: i18n, ) { this.clientId = generateClientId(); + this._i18Instance = i18Instance?.cloneInstance({ + lng: this._locale, + }); } public toString() { return `Client ${this.clientId} #${this._counter}`; } + public t(key: string, args?: any): string { + return this._i18Instance?.t(key, args) ?? key; + } + public get locale(): string|undefined { return this._locale; } diff --git a/app/server/lib/Comm.ts b/app/server/lib/Comm.ts index dac19486..185e15b8 100644 --- a/app/server/lib/Comm.ts +++ b/app/server/lib/Comm.ts @@ -51,6 +51,7 @@ import log from 'app/server/lib/log'; import {localeFromRequest} from 'app/server/lib/ServerLocale'; import {fromCallback} from 'app/server/lib/serverUtils'; import {Sessions} from 'app/server/lib/Sessions'; +import {i18n} from 'i18next'; export interface CommOptions { sessions: Sessions; // A collection of all sessions for this instance of Grist @@ -58,6 +59,7 @@ export interface CommOptions { hosts?: Hosts; // If set, we use hosts.getOrgInfo(req) to extract an organization from a (possibly versioned) url. loginMiddleware?: GristLoginMiddleware; // If set, use custom getProfile method if available httpsServer?: https.Server; // An optional HTTPS server to listen on too. + i18Instance?: i18n; // The i18next instance to use for translations. } /** @@ -229,7 +231,7 @@ export class Comm extends EventEmitter { let reuseClient = true; if (!client?.canAcceptConnection()) { reuseClient = false; - client = new Client(this, this._methods, localeFromRequest(req)); + client = new Client(this, this._methods, localeFromRequest(req), this._options.i18Instance); this._clients.set(client.clientId, client); } diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index fb01af5a..4d1a3f66 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -57,12 +57,15 @@ import {startTestingHooks} from 'app/server/lib/TestingHooks'; import {getTestLoginSystem} from 'app/server/lib/TestLogin'; import {addUploadRoute} from 'app/server/lib/uploads'; import {buildWidgetRepository, IWidgetRepository} from 'app/server/lib/WidgetRepository'; +import {readLoadedLngs, readLoadedNamespaces, setupLocale} from 'app/server/localization'; import axios from 'axios'; import * as bodyParser from 'body-parser'; import express from 'express'; import * as fse from 'fs-extra'; import * as http from 'http'; import * as https from 'https'; +import {i18n} from 'i18next'; +import i18Middleware from "i18next-http-middleware"; import mapValues = require('lodash/mapValues'); import morganLogger from 'morgan'; import {AddressInfo} from 'net'; @@ -108,6 +111,7 @@ export class FlexServer implements GristServer { public worker: DocWorkerInfo; public electronServerMethods: ElectronServerMethods; public readonly docsRoot: string; + public readonly i18Instance: i18n; private _comm: Comm; private _dbManager: HomeDBManager; private _defaultBaseDomain: string|undefined; @@ -160,6 +164,16 @@ export class FlexServer implements GristServer { this.host = process.env.GRIST_HOST || "localhost"; log.info(`== Grist version is ${version.version} (commit ${version.gitcommit})`); this.info.push(['appRoot', this.appRoot]); + // Initialize locales files. + this.i18Instance = setupLocale(this.appRoot); + if (Array.isArray(this.i18Instance.options.preload)) { + this.info.push(['i18:locale', this.i18Instance.options.preload.join(",")]); + } + if (Array.isArray(this.i18Instance.options.ns)) { + this.info.push(['i18:namespace', this.i18Instance.options.ns.join(",")]); + } + // Add language detection middleware. + this.app.use(i18Middleware.handle(this.i18Instance)); // This directory hold Grist documents. let docsRoot = path.resolve((this.options && this.options.dataDir) || process.env.GRIST_DATA_DIR || @@ -467,6 +481,10 @@ export class FlexServer implements GristServer { })); const staticApp = express.static(getAppPathTo(this.appRoot, 'static'), options); const bowerApp = express.static(getAppPathTo(this.appRoot, 'bower_components'), options); + if (process.env.GRIST_LOCALES_DIR) { + const locales = express.static(process.env.GRIST_LOCALES_DIR, options); + this.app.use("/locales", this.tagChecker.withTag(locales)); + } this.app.use(this.tagChecker.withTag(staticApp)); this.app.use(this.tagChecker.withTag(bowerApp)); } @@ -894,6 +912,7 @@ export class FlexServer implements GristServer { hosts: this._hosts, loginMiddleware: this._loginMiddleware, httpsServer: this.httpsServer, + i18Instance: this.i18Instance }); } /** @@ -1255,7 +1274,10 @@ export class FlexServer implements GristServer { } public getGristConfig(): GristLoadConfig { - return makeGristConfig(this.getDefaultHomeUrl(), {}, this._defaultBaseDomain); + return makeGristConfig(this.getDefaultHomeUrl(), { + supportedLngs: readLoadedLngs(this.i18Instance), + namespaces: readLoadedNamespaces(this.i18Instance), + }, this._defaultBaseDomain); } /** diff --git a/app/server/lib/sendAppPage.ts b/app/server/lib/sendAppPage.ts index be49a855..e961b9e8 100644 --- a/app/server/lib/sendAppPage.ts +++ b/app/server/lib/sendAppPage.ts @@ -5,6 +5,7 @@ import {isAnonymousUser, RequestWithLogin} from 'app/server/lib/Authorizer'; import {RequestWithOrg} from 'app/server/lib/extractOrg'; import {GristServer} from 'app/server/lib/GristServer'; import {getSupportedEngineChoices} from 'app/server/lib/serverUtils'; +import {readLoadedLngs, readLoadedNamespaces} from 'app/server/localization'; import * as express from 'express'; import * as fse from 'fs-extra'; import jsesc from 'jsesc'; @@ -56,6 +57,8 @@ export function makeGristConfig(homeUrl: string|null, extra: PartialAuthentication is not enforced" : ""; + // Preload all languages that will be used or are requested by client. + const preloads = req.languages.map((lng) => + readLoadedNamespaces(req.i18n).map((ns) => + `` + ).join("\n") + ).join('\n'); const content = fileContent .replace("", warning) - .replace("", getPageTitle(config)) + .replace("", getPageTitle(req, config)) .replace("", getPageMetadataHtmlSnippet(config)) .replace("", getPageTitleSuffix(server?.getGristConfig())) .replace("", `` + tagManagerSnippet) + .replace("", preloads) .replace("", customHeadHtmlSnippet) .replace( "", @@ -142,9 +152,9 @@ function configuredPageTitleSuffix() { * * Note: The string returned is escaped and safe to insert into HTML. */ -function getPageTitle(config: GristLoadConfig): string { +function getPageTitle(req: express.Request, config: GristLoadConfig): string { const maybeDoc = getDocFromConfig(config); - if (!maybeDoc) { return 'Loading...'; } + if (!maybeDoc) { return req.t('Loading') + "..."; } return handlebars.Utils.escapeExpression(maybeDoc.name); } diff --git a/app/server/localization.ts b/app/server/localization.ts new file mode 100644 index 00000000..5e06af21 --- /dev/null +++ b/app/server/localization.ts @@ -0,0 +1,75 @@ +import {lstatSync, readdirSync} from 'fs'; +import {createInstance, i18n} from 'i18next'; +import i18fsBackend from 'i18next-fs-backend'; +import {LanguageDetector} from 'i18next-http-middleware'; +import path from 'path'; + +export function setupLocale(appRoot: string): i18n { + // We are using custom instance and leave the global object intact. + const instance = createInstance(); + // By default locales are located in the appRoot folder, unless the environment variable + // GRIST_LOCALES_DIR is set. + const localeDir = process.env.GRIST_LOCALES_DIR || path.join(appRoot, 'static', 'locales'); + const supportedNamespaces: Set = new Set(); + const supportedLngs: Set = new Set(readdirSync(localeDir).map((fileName) => { + const fullPath = path.join(localeDir, fileName); + const isDirectory = lstatSync(fullPath).isDirectory(); + if (isDirectory) { + return ""; + } + const baseName = path.basename(fileName, '.json'); + const lang = baseName.split('.')[0]; + const namespace = baseName.split('.')[1]; + if (!lang || !namespace) { + throw new Error("Unrecognized resource file " + fileName); + } + supportedNamespaces.add(namespace); + return lang; + }).filter((lang) => lang)); + if (!supportedLngs.has('en') || !supportedNamespaces.has('core')) { + throw new Error("Missing core English language file"); + } + // Initialize localization filesystem plugin that will read the locale files from the localeDir. + instance.use(i18fsBackend); + // Initialize localization language detector plugin that will read the language from the request. + instance.use(LanguageDetector); + + let errorDuringLoad: Error | undefined; + instance.init({ + // Load all files synchronously. + initImmediate: false, + preload: [...supportedLngs], + supportedLngs: [...supportedLngs], + defaultNS: 'core', + ns: [...supportedNamespaces], + fallbackLng: 'en', + backend: { + loadPath: `${localeDir}/{{lng}}.{{ns}}.json` + }, + }, (err: any) => { + if (err) { + errorDuringLoad = err; + } + }).catch((err: any) => { + // This should not happen, the promise should be resolved synchronously, without + // any errors reported. + console.error("i18next failed unexpectedly", err); + }); + if (errorDuringLoad) { + throw errorDuringLoad; + } + return instance; +} + +export function readLoadedLngs(instance?: i18n): readonly string[] { + if (!instance) { return []; } + return instance?.options.preload || ['en']; +} + +export function readLoadedNamespaces(instance?: i18n): readonly string[] { + if (!instance) { return []; } + if (Array.isArray(instance?.options.ns)) { + return instance.options.ns; + } + return instance?.options.ns ? [instance.options.ns as string] : ['core']; +} diff --git a/package.json b/package.json index 0b39a498..a49ea86d 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "@types/express": "4.16.0", "@types/form-data": "2.2.1", "@types/fs-extra": "5.0.4", + "@types/i18next-fs-backend": "1.1.2", "@types/image-size": "0.0.29", "@types/js-yaml": "3.11.2", "@types/jsdom": "16.2.14", @@ -126,6 +127,9 @@ "http-proxy-agent": "5.0.0", "https-proxy-agent": "5.0.1", "i18n-iso-countries": "6.1.0", + "i18next": "21.9.1", + "i18next-fs-backend": "1.1.5", + "i18next-http-middleware": "3.2.1", "image-size": "0.6.3", "jquery": "3.5.0", "js-yaml": "3.14.1", diff --git a/static/account.html b/static/account.html index 884ab97c..2fd44b28 100644 --- a/static/account.html +++ b/static/account.html @@ -5,6 +5,7 @@ + Account<!-- INSERT TITLE SUFFIX --> diff --git a/static/activation.html b/static/activation.html index f9eb1cf1..0a0378df 100644 --- a/static/activation.html +++ b/static/activation.html @@ -5,6 +5,7 @@ + Activation<!-- INSERT TITLE SUFFIX --> diff --git a/static/app.html b/static/app.html index 3c7120c2..b346435f 100644 --- a/static/app.html +++ b/static/app.html @@ -14,6 +14,7 @@ + diff --git a/static/error.html b/static/error.html index 0a5543b8..fe838b5c 100644 --- a/static/error.html +++ b/static/error.html @@ -5,6 +5,7 @@ + Loading...<!-- INSERT TITLE SUFFIX --> diff --git a/static/locales/en.core.json b/static/locales/en.core.json new file mode 100644 index 00000000..1f1a2176 --- /dev/null +++ b/static/locales/en.core.json @@ -0,0 +1,11 @@ +{ + "Welcome": "Welcome to Grist!", + "Loading": "Loading", + "AddNew": "Add New", + "OtherSites": "Other Sites", + "OtherSitesWelcome": "You are on the {{siteName}} site. You also have access to the following sites:", + "OtherSitesWelcome_personal": "You are on your personal site. You also have access to the following sites:", + "AllDocuments": "All Documents", + "ExamplesAndTemplates": "Examples and Templates", + "MoreExamplesAndTemplates": "More Examples and Templates" +} diff --git a/test/nbrowser/Localization.ts b/test/nbrowser/Localization.ts new file mode 100644 index 00000000..6ab7d2bc --- /dev/null +++ b/test/nbrowser/Localization.ts @@ -0,0 +1,169 @@ +import * as gu from 'test/nbrowser/gristUtils'; +import {server, setupTestSuite} from 'test/nbrowser/testUtils'; +import {assert, driver} from 'mocha-webdriver'; +import * as testUtils from 'test/server/testUtils'; +import {getAppRoot} from 'app/server/lib/places'; +import fetch from "node-fetch"; +import fs from "fs"; +import os from "os"; +import path from 'path'; + +describe("Localization", function() { + this.timeout(20000); + setupTestSuite(); + + before(async function() { + const session = await gu.session().personalSite.anon.login(); + await session.loadRelPath("/"); + }); + + it("uses default options for English language", async function() { + // Currently, there is not much translated, so test just what we have. + assert.equal(await driver.findWait('.test-welcome-title', 3000).getText(), 'Welcome to Grist!'); + // Grist config should contain the list of supported languages; + const gristConfig: any = await driver.executeScript("return window.gristConfig"); + + // core and en is required. + assert.isTrue(gristConfig.namespaces.includes("core")); + assert.isTrue(gristConfig.supportedLngs.includes("en")); + }); + + it("loads all files from resource folder", async function() { + if (server.isExternalServer()) { + this.skip(); + } + // Grist config should contain the list of supported languages; + const gristConfig: any = await driver.executeScript("return window.gristConfig"); + // Should report all supported languages and namespaces. + const localeDirectory = path.join(getAppRoot(), 'static', 'locales'); + // Read all file names from localeDirectory + const langs: Set = new Set(); + const namespaces: Set = new Set(); + for (const file of fs.readdirSync(localeDirectory)) { + if (file.endsWith(".json")) { + const lang = file.split('.')[0]; + const ns = file.split('.')[1]; + langs.add(lang); + namespaces.add(ns); + } + } + assert.deepEqual(gristConfig.supportedLngs.sort(), [...langs].sort()); + assert.deepEqual(gristConfig.namespaces.sort(), [...namespaces].sort()); + }); + + // Now make a Polish language file, and test that it is used. + describe("with Polish language file", function() { + let oldEnv: testUtils.EnvironmentSnapshot; + let tempLocale: string; + before(async function() { + if (server.isExternalServer()) { + this.skip(); + } + oldEnv = new testUtils.EnvironmentSnapshot(); + // Add another language to the list of supported languages. + tempLocale = makeCopy(); + createLanguage(tempLocale, "pl"); + process.env.GRIST_LOCALES_DIR = tempLocale; + await server.restart(); + }); + + after(async () => { + oldEnv.restore(); + await server.restart(); + }); + + it("detects correct language from client headers", async function() { + const homeUrl = `${server.getHost()}/o/docs`; + // Read response from server, and check that it contains the correct language. + const enResponse = await (await fetch(homeUrl)).text(); + const plResponse = await (await fetch(homeUrl, {headers: {"Accept-Language": "pl-PL,pl;q=1"}})).text(); + const ptResponse = await (await fetch(homeUrl, {headers: {"Accept-Language": "pt-PR,pt;q=1"}})).text(); + + function present(response: string, ...langs: string[]) { + for (const lang of langs) { + assert.include(response, `href="locales/${lang}.core.json"`); + } + } + + function notPresent(response: string, ...langs: string[]) { + for (const lang of langs) { + assert.notInclude(response, `href="locales/${lang}.core.json"`); + } + } + + // English locale is preloaded always. + present(enResponse, "en"); + present(plResponse, "en"); + present(ptResponse, "en"); + + // Other locales are not preloaded for English. + notPresent(enResponse, "pl", "pl-PL", "en-US"); + + // For Polish we have additional pl locale. + present(plResponse, "pl"); + // But only pl code is preloaded. + notPresent(plResponse, "pl-PL"); + + // For Portuguese we have only en. + notPresent(ptResponse, "pt", "pt-PR", "pl", "en-US"); + }); + + it("loads correct languages from file system", async function() { + modifyByCode(tempLocale, "en", {Welcome: 'TestMessage'}); + await driver.navigate().refresh(); + assert.equal(await driver.findWait('.test-welcome-title', 3000).getText(), 'TestMessage'); + const gristConfig: any = await driver.executeScript("return window.gristConfig"); + assert.deepEqual(gristConfig.supportedLngs, ['en', 'pl']); + }); + }); + + it("breaks the server if something is wrong with resource files", async () => { + const oldEnv = new testUtils.EnvironmentSnapshot(); + try { + // Wrong path to locales. + process.env.GRIST_LOCALES_DIR = __filename; + await assert.isRejected(server.restart()); + // Empty folder. + const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'grist_test_')); + process.env.GRIST_LOCALES_DIR = tempDirectory; + await assert.isRejected(server.restart()); + // Wrong file format. + fs.writeFileSync(path.join(tempDirectory, 'dummy.json'), 'invalid json'); + await assert.isRejected(server.restart()); + } finally { + oldEnv.restore(); + await server.restart(); + } + }); + + /** + * Creates a new language by coping existing "en" resources. + */ + function createLanguage(localesPath: string, code: string) { + for (const file of fs.readdirSync(localesPath)) { + const newFile = file.replace('en', code); + fs.copyFileSync(path.join(localesPath, file), path.join(localesPath, newFile)); + } + } + + /** + * Makes a copy of all resource files and returns path to the temporary directory. + */ + function makeCopy() { + const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'grist_test_')); + const localeDirectory = path.join(getAppRoot(), 'static', 'locales'); + // Copy all files from localeDirectory to tempDirectory. + fs.readdirSync(localeDirectory).forEach(file => { + fs.copyFileSync(path.join(localeDirectory, file), path.join(tempDirectory, file)); + }); + return tempDirectory; + } + + function modifyByCode(localeDir: string, code: string, obj: any) { + // Read current core localization file. + const filePath = path.join(localeDir, `${code}.core.json`); + const resources = JSON.parse(fs.readFileSync(filePath).toString()); + const newResource = Object.assign(resources, obj); + fs.writeFileSync(filePath, JSON.stringify(newResource)); + } +}); diff --git a/yarn.lock b/yarn.lock index 0988a348..f3c4964b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,13 @@ # yarn lockfile v1 +"@babel/runtime@^7.17.2": + version "7.19.0" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.19.0.tgz#22b11c037b094d27a8a2504ea4dcff00f50e2259" + integrity sha512-eR8Lo9hnDS7tqkO7NsV+mKvCmv5boaXFSZ70DnfhcgiEne8hv9oCEd36Klw74EtizEqLsy4YnW8UWwpBVolHZA== + dependencies: + regenerator-runtime "^0.13.4" + "@discoveryjs/json-ext@^0.5.0": version "0.5.7" resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70" @@ -383,6 +390,13 @@ dependencies: "@types/node" "*" +"@types/i18next-fs-backend@1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@types/i18next-fs-backend/-/i18next-fs-backend-1.1.2.tgz#4f3116769229371fcdf64bbdb6841ea745e392f9" + integrity sha512-ZzTRXA5B0x0oGhzKNp08IsYjZpli4LjRZpg3q4j0XFxN5lKG2MVLnR4yHX8PPExBk4sj9Yfk1z9O6CjPrAlmIQ== + dependencies: + i18next "^21.0.1" + "@types/image-size@0.0.29": version "0.0.29" resolved "https://registry.yarnpkg.com/@types/image-size/-/image-size-0.0.29.tgz#0924d4ec95edc82f615b7f634cba31534b81c351" @@ -3614,6 +3628,23 @@ i18n-iso-countries@6.1.0: dependencies: diacritics "1.3.0" +i18next-fs-backend@1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/i18next-fs-backend/-/i18next-fs-backend-1.1.5.tgz#dcbd8227b1c1e4323b3c40e269d4762e8313d9e5" + integrity sha512-raTel3EfshiUXxR0gvmIoqp75jhkj8+7R1LjB006VZKPTFBbXyx6TlUVhb8Z9+7ahgpFbcQg1QWVOdf/iNzI5A== + +i18next-http-middleware@3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/i18next-http-middleware/-/i18next-http-middleware-3.2.1.tgz#a0dff150de2273ec650da67336ad882eef58d179" + integrity sha512-zBwXxDChT0YLoTXIR6jRuqnUUhXW0Iw7egoTnNXyaDRtTbfWNXwU0a53ThyuRPQ+k+tXu3ZMNKRzfLuononaRw== + +i18next@21.9.1, i18next@^21.0.1: + version "21.9.1" + resolved "https://registry.yarnpkg.com/i18next/-/i18next-21.9.1.tgz#9e3428990f5b2cc9ac1b98dd025f3e411c368249" + integrity sha512-ITbDrAjbRR73spZAiu6+ex5WNlHRr1mY+acDi2ioTHuUiviJqSz269Le1xHAf0QaQ6GgIHResUhQNcxGwa/PhA== + dependencies: + "@babel/runtime" "^7.17.2" + iconv-lite@0.4.23: version "0.4.23" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" @@ -5819,6 +5850,11 @@ reflect-metadata@^0.1.13: resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08" integrity sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg== +regenerator-runtime@^0.13.4: + version "0.13.9" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52" + integrity sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA== + registry-auth-token@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/registry-auth-token/-/registry-auth-token-4.1.1.tgz#40a33be1e82539460f94328b0f7f0f84c16d9479"