(core) i18

Summary:
Adding initial work for localization support.

Summary in https://grist.quip.com/OtZKA6RHdQ6T/Internationalization-and-Localization

Test Plan: Not yet

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: paulfitz

Differential Revision: https://phab.getgrist.com/D3633

app/server/lib/Client.ts

@@ -16,6 +16,7 @@ import log from 'app/server/lib/log';
 import {LogMethods} from "app/server/lib/LogMethods";
 import {shortDesc} from 'app/server/lib/shortDesc';
 import {fromCallback} from 'app/server/lib/serverUtils';
+import {i18n} from 'i18next';
 import * as crypto from 'crypto';
 import moment from 'moment';
 import * as WebSocket from 'ws';
@@ -92,17 +93,26 @@ export class Client {
 
   // Identifier for the current GristWSConnection object connected to this client.
   private _counter: string|null = null;
+  private _i18Instance?: i18n;
 
   constructor(
     private _comm: Comm,
     private _methods: Map<string, ClientMethod>,
     private _locale: string,
+    i18Instance?: i18n,
   ) {
     this.clientId = generateClientId();
+    this._i18Instance = i18Instance?.cloneInstance({
+      lng: this._locale,
+    });
   }
 
   public toString() { return `Client ${this.clientId} #${this._counter}`; }
 
+  public t(key: string, args?: any): string {
+    return this._i18Instance?.t(key, args) ?? key;
+  }
+
   public get locale(): string|undefined {
     return this._locale;
   }

app/server/lib/Comm.ts

@@ -51,6 +51,7 @@ import log from 'app/server/lib/log';
 import {localeFromRequest} from 'app/server/lib/ServerLocale';
 import {fromCallback} from 'app/server/lib/serverUtils';
 import {Sessions} from 'app/server/lib/Sessions';
+import {i18n} from 'i18next';
 
 export interface CommOptions {
   sessions: Sessions;                   // A collection of all sessions for this instance of Grist
@@ -58,6 +59,7 @@ export interface CommOptions {
   hosts?: Hosts;  // If set, we use hosts.getOrgInfo(req) to extract an organization from a (possibly versioned) url.
   loginMiddleware?: GristLoginMiddleware; // If set, use custom getProfile method if available
   httpsServer?: https.Server;   // An optional HTTPS server to listen on too.
+  i18Instance?: i18n;           // The i18next instance to use for translations.
 }
 
 /**
@@ -229,7 +231,7 @@ export class Comm extends EventEmitter {
     let reuseClient = true;
     if (!client?.canAcceptConnection()) {
       reuseClient = false;
-      client = new Client(this, this._methods, localeFromRequest(req));
+      client = new Client(this, this._methods, localeFromRequest(req), this._options.i18Instance);
       this._clients.set(client.clientId, client);
     }
 

app/server/lib/FlexServer.ts

@@ -57,12 +57,15 @@ import {startTestingHooks} from 'app/server/lib/TestingHooks';
 import {getTestLoginSystem} from 'app/server/lib/TestLogin';
 import {addUploadRoute} from 'app/server/lib/uploads';
 import {buildWidgetRepository, IWidgetRepository} from 'app/server/lib/WidgetRepository';
+import {readLoadedLngs, readLoadedNamespaces, setupLocale} from 'app/server/localization';
 import axios from 'axios';
 import * as bodyParser from 'body-parser';
 import express from 'express';
 import * as fse from 'fs-extra';
 import * as http from 'http';
 import * as https from 'https';
+import {i18n} from 'i18next';
+import i18Middleware from "i18next-http-middleware";
 import mapValues = require('lodash/mapValues');
 import morganLogger from 'morgan';
 import {AddressInfo} from 'net';
@@ -108,6 +111,7 @@ export class FlexServer implements GristServer {
   public worker: DocWorkerInfo;
   public electronServerMethods: ElectronServerMethods;
   public readonly docsRoot: string;
+  public readonly i18Instance: i18n;
   private _comm: Comm;
   private _dbManager: HomeDBManager;
   private _defaultBaseDomain: string|undefined;
@@ -160,6 +164,16 @@ export class FlexServer implements GristServer {
     this.host = process.env.GRIST_HOST || "localhost";
     log.info(`== Grist version is ${version.version} (commit ${version.gitcommit})`);
     this.info.push(['appRoot', this.appRoot]);
+    // Initialize locales files.
+    this.i18Instance = setupLocale(this.appRoot);
+    if (Array.isArray(this.i18Instance.options.preload)) {
+      this.info.push(['i18:locale', this.i18Instance.options.preload.join(",")]);
+    }
+    if (Array.isArray(this.i18Instance.options.ns)) {
+      this.info.push(['i18:namespace', this.i18Instance.options.ns.join(",")]);
+    }
+    // Add language detection middleware.
+    this.app.use(i18Middleware.handle(this.i18Instance));
     // This directory hold Grist documents.
     let docsRoot = path.resolve((this.options && this.options.dataDir) ||
                                   process.env.GRIST_DATA_DIR ||
@@ -467,6 +481,10 @@ export class FlexServer implements GristServer {
     }));
     const staticApp = express.static(getAppPathTo(this.appRoot, 'static'), options);
     const bowerApp = express.static(getAppPathTo(this.appRoot, 'bower_components'), options);
+    if (process.env.GRIST_LOCALES_DIR) {
+      const locales = express.static(process.env.GRIST_LOCALES_DIR, options);
+      this.app.use("/locales", this.tagChecker.withTag(locales));
+    }
     this.app.use(this.tagChecker.withTag(staticApp));
     this.app.use(this.tagChecker.withTag(bowerApp));
   }
@@ -894,6 +912,7 @@ export class FlexServer implements GristServer {
       hosts: this._hosts,
       loginMiddleware: this._loginMiddleware,
       httpsServer: this.httpsServer,
+      i18Instance: this.i18Instance
     });
   }
   /**
@@ -1255,7 +1274,10 @@ export class FlexServer implements GristServer {
   }
 
   public getGristConfig(): GristLoadConfig {
-    return makeGristConfig(this.getDefaultHomeUrl(), {}, this._defaultBaseDomain);
+    return makeGristConfig(this.getDefaultHomeUrl(), {
+      supportedLngs: readLoadedLngs(this.i18Instance),
+      namespaces: readLoadedNamespaces(this.i18Instance),
+    }, this._defaultBaseDomain);
   }
 
   /**

app/server/lib/sendAppPage.ts

@@ -5,6 +5,7 @@ import {isAnonymousUser, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {RequestWithOrg} from 'app/server/lib/extractOrg';
 import {GristServer} from 'app/server/lib/GristServer';
 import {getSupportedEngineChoices} from 'app/server/lib/serverUtils';
+import {readLoadedLngs, readLoadedNamespaces} from 'app/server/localization';
 import * as express from 'express';
 import * as fse from 'fs-extra';
 import jsesc from 'jsesc';
@@ -56,6 +57,8 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadCo
     tagManagerId: process.env.GOOGLE_TAG_MANAGER_ID,
     activation: getActivation(req as RequestWithLogin | undefined),
     enableCustomCss: process.env.APP_STATIC_INCLUDE_CUSTOM_CSS === 'true',
+    supportedLngs: readLoadedLngs(req?.i18n),
+    namespaces: readLoadedNamespaces(req?.i18n),
     ...extra,
   };
 }
@@ -101,12 +104,19 @@ export function makeSendAppPage(opts: {
     const staticBaseUrl = `${staticOrigin}/v/${options.tag || tag}/`;
     const customHeadHtmlSnippet = server?.create.getExtraHeadHtml?.() ?? "";
     const warning = testLogin ? "<div class=\"dev_warning\">Authentication is not enforced</div>" : "";
+    // Preload all languages that will be used or are requested by client.
+    const preloads = req.languages.map((lng) =>
+      readLoadedNamespaces(req.i18n).map((ns) =>
+        `<link rel="preload" href="locales/${lng}.${ns}.json" as="fetch" type="application/json" crossorigin>`
+      ).join("\n")
+    ).join('\n');
     const content = fileContent
       .replace("<!-- INSERT WARNING -->", warning)
-      .replace("<!-- INSERT TITLE -->", getPageTitle(config))
+      .replace("<!-- INSERT TITLE -->", getPageTitle(req, config))
       .replace("<!-- INSERT META -->", getPageMetadataHtmlSnippet(config))
       .replace("<!-- INSERT TITLE SUFFIX -->", getPageTitleSuffix(server?.getGristConfig()))
       .replace("<!-- INSERT BASE -->", `<base href="${staticBaseUrl}">` + tagManagerSnippet)
+      .replace("<!-- INSERT LOCALE -->", preloads)
       .replace("<!-- INSERT CUSTOM -->", customHeadHtmlSnippet)
       .replace(
         "<!-- INSERT CONFIG -->",
@@ -142,9 +152,9 @@ function configuredPageTitleSuffix() {
  *
  * Note: The string returned is escaped and safe to insert into HTML.
  */
-function getPageTitle(config: GristLoadConfig): string {
+function getPageTitle(req: express.Request, config: GristLoadConfig): string {
   const maybeDoc = getDocFromConfig(config);
-  if (!maybeDoc) { return 'Loading...'; }
+  if (!maybeDoc) { return req.t('Loading') + "..."; }
 
   return handlebars.Utils.escapeExpression(maybeDoc.name);
 }

app/server/localization.ts

@@ -0,0 +1,75 @@
+import {lstatSync, readdirSync} from 'fs';
+import {createInstance, i18n} from 'i18next';
+import i18fsBackend from 'i18next-fs-backend';
+import {LanguageDetector} from 'i18next-http-middleware';
+import path from 'path';
+
+export function setupLocale(appRoot: string): i18n {
+  // We are using custom instance and leave the global object intact.
+  const instance = createInstance();
+  // By default locales are located in the appRoot folder, unless the environment variable
+  // GRIST_LOCALES_DIR is set.
+  const localeDir = process.env.GRIST_LOCALES_DIR || path.join(appRoot, 'static', 'locales');
+  const supportedNamespaces: Set<string> = new Set();
+  const supportedLngs: Set<string> = new Set(readdirSync(localeDir).map((fileName) => {
+    const fullPath = path.join(localeDir, fileName);
+    const isDirectory = lstatSync(fullPath).isDirectory();
+    if (isDirectory) {
+      return "";
+    }
+    const baseName = path.basename(fileName, '.json');
+    const lang = baseName.split('.')[0];
+    const namespace = baseName.split('.')[1];
+    if (!lang || !namespace) {
+      throw new Error("Unrecognized resource file " + fileName);
+    }
+    supportedNamespaces.add(namespace);
+    return lang;
+  }).filter((lang) => lang));
+  if (!supportedLngs.has('en') || !supportedNamespaces.has('core')) {
+    throw new Error("Missing core English language file");
+  }
+  // Initialize localization filesystem plugin that will read the locale files from the localeDir.
+  instance.use(i18fsBackend);
+  // Initialize localization language detector plugin that will read the language from the request.
+  instance.use(LanguageDetector);
+
+  let errorDuringLoad: Error | undefined;
+  instance.init({
+    // Load all files synchronously.
+    initImmediate: false,
+    preload: [...supportedLngs],
+    supportedLngs: [...supportedLngs],
+    defaultNS: 'core',
+    ns: [...supportedNamespaces],
+    fallbackLng: 'en',
+    backend: {
+      loadPath: `${localeDir}/{{lng}}.{{ns}}.json`
+    },
+  }, (err: any) => {
+    if (err) {
+      errorDuringLoad = err;
+    }
+  }).catch((err: any) => {
+    // This should not happen, the promise should be resolved synchronously, without
+    // any errors reported.
+    console.error("i18next failed unexpectedly", err);
+  });
+  if (errorDuringLoad) {
+    throw errorDuringLoad;
+  }
+  return instance;
+}
+
+export function readLoadedLngs(instance?: i18n): readonly string[] {
+  if (!instance) { return []; }
+  return instance?.options.preload || ['en'];
+}
+
+export function readLoadedNamespaces(instance?: i18n): readonly string[] {
+  if (!instance) { return []; }
+  if (Array.isArray(instance?.options.ns)) {
+    return instance.options.ns;
+  }
+  return instance?.options.ns ? [instance.options.ns as string] : ['core'];
+}