(core) Extending default locale list

Summary: Adding more locale codes to support more countries in document settings

Test Plan: existing tests

Reviewers: dsagal

Reviewed By: dsagal

Subscribers: dsagal

Differential Revision: https://phab.getgrist.com/D3018

app/client/ui/DocumentSettings.ts

@@ -14,13 +14,12 @@ import {DocPageModel} from 'app/client/models/DocPageModel';
 import {testId, vars} from 'app/client/ui2018/cssVars';
 import {select} from 'app/client/ui2018/menus';
 import {saveModal} from 'app/client/ui2018/modals';
+import {buildCurrencyPicker} from 'app/client/widgets/CurrencyPicker';
 import {buildTZAutocomplete} from 'app/client/widgets/TZAutocomplete';
 import {EngineCode} from 'app/common/DocumentSettings';
 import {GristLoadConfig} from 'app/common/gristUrls';
-import {locales} from "app/common/Locales";
+import {propertyCompare} from "app/common/gutil";
-import {buildCurrencyPicker} from 'app/client/widgets/CurrencyPicker';
+import {getCurrency, locales} from "app/common/Locales";
-import * as LocaleCurrency from 'locale-currency';
-
 /**
  * Builds a simple saveModal for saving settings.
  */
@@ -50,7 +49,7 @@ export async function showDocSettingsModal(docInfo: DocInfoRec, docPageModel: Do
         cssDataRow('Currency:'),
         cssDataRow(dom.domComputed(localeObs, (l) =>
           dom.create(buildCurrencyPicker, currencyObs, (val) => currencyObs.set(val),
-            {defaultCurrencyLabel: `Local currency (${LocaleCurrency.getCurrency(l)})`})
+            {defaultCurrencyLabel: `Local currency (${getCurrency(l)})`})
         )),
         canChangeEngine ? [
           cssDataRow('Engine:'),
@@ -101,7 +100,7 @@ function buildLocaleSelect(
     label: l.name,
     locale: l.code,
     cleanText: l.name.trim().toLowerCase(),
-  }));
+  })).sort(propertyCompare("label"));
   const acIndex = new ACIndexImpl<LocaleItem>(localeList, 200, true);
   // AC select will show the value (in this case locale) not a label when something is selected.
   // To show the label - create another observable that will be in sync with the value, but

app/common/LocaleCodes.ts

@@ -0,0 +1,36 @@
+// This file was generated by core/buildtools/generate_locale_list.js at 2021-09-23T08:32:25.517Z
+export const localeCodes = [
+  "af-ZA", "ak-GH", "am-ET", "ar-AE", "ar-BH", "ar-DZ", "ar-EG",
+  "ar-IN", "ar-IQ", "ar-JO", "ar-KW", "ar-LB", "ar-LY", "ar-MA",
+  "ar-OM", "ar-QA", "ar-SA", "ar-SD", "ar-SS", "ar-SY", "ar-TN",
+  "ar-YE", "as-IN", "ast-ES", "az-AZ", "az-IR", "be-BY", "bem-ZM",
+  "bg-BG", "bn-BD", "bn-IN", "bo-CN", "bo-IN", "br-FR", "brx-IN",
+  "bs-BA", "ca-AD", "ca-ES", "ca-FR", "ca-IT", "ce-RU", "chr-US",
+  "ckb-IQ", "cs-CZ", "cy-GB", "da-DK", "de-AT", "de-BE", "de-CH",
+  "de-DE", "de-IT", "de-LI", "de-LU", "doi-IN", "dz-BT", "el-CY",
+  "el-GR", "en-AG", "en-AU", "en-BW", "en-CA", "en-DK", "en-GB",
+  "en-HK", "en-IE", "en-IL", "en-IN", "en-NG", "en-NZ", "en-PH",
+  "en-SC", "en-SG", "en-US", "en-ZA", "en-ZM", "en-ZW", "es-AR",
+  "es-BO", "es-CL", "es-CO", "es-CR", "es-CU", "es-DO", "es-EC",
+  "es-ES", "es-GT", "es-HN", "es-MX", "es-NI", "es-PA", "es-PE",
+  "es-PR", "es-PY", "es-SV", "es-US", "es-UY", "es-VE", "et-EE",
+  "eu-ES", "fa-IR", "ff-SN", "fi-FI", "fil-PH", "fo-FO", "fr-BE",
+  "fr-CA", "fr-CH", "fr-FR", "fr-LU", "fur-IT", "fy-DE", "fy-NL",
+  "ga-IE", "gd-GB", "gl-ES", "gu-IN", "gv-GB", "ha-NG", "he-IL",
+  "hi-IN", "hr-HR", "hsb-DE", "hu-HU", "hy-AM", "ia-FR", "id-ID",
+  "ig-NG", "is-IS", "it-CH", "it-IT", "ja-JP", "kab-DZ", "ka-GE",
+  "kk-KZ", "kl-GL", "km-KH", "kn-IN", "kok-IN", "ko-KR", "ks-IN",
+  "ku-TR", "kw-GB", "ky-KG", "lb-LU", "lg-UG", "ln-CD", "lo-LA",
+  "lt-LT", "lv-LV", "mai-IN", "mai-NP", "mfe-MU", "mg-MG", "mi-NZ",
+  "mk-MK", "ml-IN", "mni-IN", "mn-MN", "mr-IN", "ms-MY", "mt-MT",
+  "my-MM", "ne-NP", "nl-AW", "nl-BE", "nl-NL", "nn-NO", "om-ET",
+  "om-KE", "or-IN", "os-RU", "pa-IN", "pa-PK", "pl-PL", "ps-AF",
+  "pt-BR", "pt-PT", "ro-RO", "ru-RU", "ru-UA", "rw-RW", "sa-IN",
+  "sat-IN", "sd-IN", "se-NO", "si-LK", "sk-SK", "sl-SI", "so-DJ",
+  "so-ET", "so-KE", "so-SO", "sq-AL", "sq-MK", "sr-ME", "sr-RS",
+  "sv-FI", "sv-SE", "sw-KE", "sw-TZ", "ta-IN", "ta-LK", "te-IN",
+  "tg-TJ", "th-TH", "ti-ER", "ti-ET", "tk-TM", "to-TO", "tr-CY",
+  "tr-TR", "tt-RU", "ug-CN", "uk-UA", "ur-IN", "ur-PK", "uz-UZ",
+  "vi-VN", "wae-CH", "wo-SN", "xh-ZA", "yi-US", "yo-NG", "yue-HK",
+  "zh-CN", "zh-HK", "zh-SG", "zh-TW", "zu-ZA"
+];

app/common/Locales.ts

@@ -1,16 +1,9 @@
-import LocaleCurrency = require('locale-currency/map');
+import * as LocaleCurrencyMap from 'locale-currency/map';
+import * as LocaleCurrency from 'locale-currency';
 import {nativeCompare} from 'app/common/gutil';
+import {localeCodes} from "app/common/LocaleCodes";
 
-const localeCodes = [
+const DEFAULT_CURRENCY = "USD";
-  "es-AR", "hy-AM", "en-AU", "az-AZ", "be-BY", "quz-BO", "pt-BR",
-  "bg-BG", "en-CA", "arn-CL", "es-CO", "hr-HR", "cs-CZ", "da-DK",
-  "es-EC", "ar-EG", "fi-FI", "fr-FR", "ka-GE", "de-DE", "el-GR", "en-HK",
-  "hu-HU", "hi-IN", "id-ID", "ga-IE", "ar-IL", "it-IT", "ja-JP", "kk-KZ",
-  "lv-LV", "lt-LT", "es-MX", "mn-MN", "my-MM", "nl-NL", "nb-NO",
-  "es-PY", "ceb-PH", "pl-PL", "pt-PT", "ro-RO", "ru-RU", "sr-RS",
-  "sk-SK", "sl-SI", "ko-KR", "es-ES", "sv-SE", "de-CH", "zh-TW", "th-TH",
-  "tr-TR", "uk-UA", "en-GB", "en-US", "es-UY", "es-VE", "vi-VN"
-];
 
 export interface Locale {
   name: string;
@@ -21,9 +14,23 @@ export let locales: Readonly<Locale[]>;
 
 // Intl.DisplayNames is only supported on recent browsers, so proceed with caution.
 try {
-  const localeDisplay = new Intl.DisplayNames('en', {type: 'region'});
+  const regionDisplay = new Intl.DisplayNames('en', {type: 'region'});
-  locales = localeCodes.map(code => {
+  const languageDisplay = new Intl.DisplayNames('en', {type: 'language'});
-    return { name: localeDisplay.of(new Intl.Locale(code).region), code };
+  const display = (code: string) => {
+    try {
+      const locale = new Intl.Locale(code);
+      const regionName = regionDisplay.of(locale.region);
+      const languageName = languageDisplay.of(locale.language);
+      return `${regionName} (${languageName})`;
+    } catch (ex) {
+      return code;
+    }
+  };
+  // Leave only those that are supported by current system (can be translated to human readable form).
+  // Though, this file is in common, it is safe to filter by current system
+  // as the list should be already filtered by codes that are supported by the backend.
+  locales = Intl.DisplayNames.supportedLocalesOf(localeCodes).map(code => {
+    return {name: display(code), code};
   });
 } catch {
   // Fall back to using the locale code as the display name.
@@ -37,15 +44,25 @@ export interface Currency {
 
 export let currencies: Readonly<Currency[]>;
 
+// locale-currency package doesn't have South Sudanese pound currency or a default value for Kosovo
+LocaleCurrencyMap["SS"] = "SSP";
+LocaleCurrencyMap["XK"] = "EUR";
+const currenciesCodes = Object.values(LocaleCurrencyMap);
+export function getCurrency(code: string) {
+  const currency = LocaleCurrency.getCurrency(code);
+  // Fallback to USD
+  return currency ?? DEFAULT_CURRENCY;
+}
+
 // Intl.DisplayNames is only supported on recent browsers, so proceed with caution.
 try {
   const currencyDisplay = new Intl.DisplayNames('en', {type: 'currency'});
-  currencies = [...new Set(Object.values(LocaleCurrency))].map(code => {
+  currencies = [...new Set(currenciesCodes)].map(code => {
     return {name: currencyDisplay.of(code), code};
   });
 } catch {
   // Fall back to using the currency code as the display name.
-  currencies = [...new Set(Object.values(LocaleCurrency))].map(code => {
+  currencies = [...new Set(currenciesCodes)].map(code => {
     return {name: code, code};
   });
 }

app/common/declarations.d.ts

@@ -12,12 +12,14 @@ declare module "locale-currency/map" {
 
 declare namespace Intl {
   class DisplayNames {
+    public static supportedLocalesOf(locales: string | string[]): string[];
     constructor(locales?: string, options?: object);
     public of(code: string): string;
   }
 
   class Locale {
     public region: string;
+    public language: string;
     constructor(locale: string);
   }
 }

app/common/gutil.ts

@@ -487,6 +487,15 @@ export function nativeCompare<T>(a: T, b: T): number {
   return (a < b ? -1 : (a > b ? 1 : 0));
 }
 
+/**
+ * Creates a function that compares objects by a property value.
+ */
+export function propertyCompare<T>(property: keyof T) {
+  return function(a: T, b: T) {
+    return nativeCompare(a[property], b[property]);
+  };
+}
+
 // TODO: In the future, locale should be a value associated with the document or the user.
 export const defaultLocale = 'en-US';
 export const defaultCollator = new Intl.Collator(defaultLocale);

buildtools/generate_locale_list.js

@@ -0,0 +1,152 @@
+/**
+ * This file generates list of supported locales for DocumentSettings editor,
+ * stored in core/app/common/LocaleCodes.ts
+ *
+ * To regenerate the list run this script on linux (should be the run on the same OS as the one
+ * being use to host Grist):
+ *  node generate_locale_list.js
+ *
+ * Full list of codes was taken from https://lh.2xlibre.net/locales/.
+ * List was modified by:
+ * - removing tl-PH (PHILIPPINES Tagalog) as it is translated the same in English as Filipino.
+ * - changing KV (Kosovo) to XK as this one is supported on Chrome (also in Node, Firefox and Python)
+ *
+ * For node:
+ * - List of supported locales is generated by Intl.DisplayNames.supportedLocalesOf(locale) method.
+ *
+ * For python
+ * - List is generated by using locale module (on Ubuntu 20.04), by script:
+ *  import locale
+ *  print(", ".join(set([('"' + l.replace("_", "-").split(".")[0] + '"') for l in locale.locale_alias.values()])))
+ */
+
+// Locale codes from https://lh.2xlibre.net/locales/
+const localeCodes = [
+  "aa-DJ", "aa-ER", "aa-ET", "af-ZA", "agr-PE", "ak-GH", "am-ET",
+  "an-ES", "anp-IN", "ar-AE", "ar-BH", "ar-DZ", "ar-EG", "ar-IN",
+  "ar-IQ", "ar-JO", "ar-KW", "ar-LB", "ar-LY", "ar-MA", "ar-OM",
+  "ar-QA", "ar-SA", "ar-SD", "ar-SS", "ar-SY", "ar-TN", "ar-YE",
+  "as-IN", "ast-ES", "ayc-PE", "az-AZ", "az-IR", "be-BY", "bem-ZM",
+  "ber-DZ", "ber-MA", "bg-BG", "bhb-IN", "bho-IN", "bho-NP", "bi-VU",
+  "bn-BD", "bn-IN", "bo-CN", "bo-IN", "br-FR", "brx-IN", "bs-BA",
+  "byn-ER", "ca-AD", "ca-ES", "ca-FR", "ca-IT", "ce-RU", "chr-US",
+  "ckb-IQ", "cmn-TW", "crh-UA", "csb-PL", "cs-CZ", "cv-RU", "cy-GB",
+  "da-DK", "de-AT", "de-BE", "de-CH", "de-DE", "de-IT", "de-LI",
+  "de-LU", "doi-IN", "dsb-DE", "dv-MV", "dz-BT", "el-CY", "el-GR",
+  "en-AG", "en-AU", "en-BW", "en-CA", "en-DK", "en-GB", "en-HK",
+  "en-IE", "en-IL", "en-IN", "en-NG", "en-NZ", "en-PH", "en-SC",
+  "en-SG", "en-US", "en-ZA", "en-ZM", "en-ZW", "es-AR", "es-BO",
+  "es-CL", "es-CO", "es-CR", "es-CU", "es-DO", "es-EC", "es-ES",
+  "es-GT", "es-HN", "es-MX", "es-NI", "es-PA", "es-PE", "es-PR",
+  "es-PY", "es-SV", "es-US", "es-UY", "es-VE", "et-EE", "eu-ES",
+  "fa-IR", "ff-SN", "fi-FI", "fil-PH", "fo-FO", "fr-BE", "fr-CA",
+  "fr-CH", "fr-FR", "fr-LU", "fur-IT", "fy-DE", "fy-NL", "ga-IE",
+  "gd-GB", "gez-ER", "gez-ET", "gl-ES", "gu-IN", "gv-GB", "hak-TW",
+  "ha-NG", "he-IL", "hif-FJ", "hi-IN", "hne-IN", "hr-HR", "hsb-DE",
+  "ht-HT", "hu-HU", "hy-AM", "ia-FR", "id-ID", "ig-NG", "ik-CA",
+  "is-IS", "it-CH", "it-IT", "iu-CA", "ja-JP", "kab-DZ", "ka-GE",
+  "kk-KZ", "kl-GL", "km-KH", "kn-IN", "kok-IN", "ko-KR", "ks-IN",
+  "ku-TR", "kw-GB", "ky-KG", "lb-LU", "lg-UG", "li-BE", "lij-IT",
+  "li-NL", "ln-CD", "lo-LA", "lt-LT", "lv-LV", "lzh-TW", "mag-IN",
+  "mai-IN", "mai-NP", "mfe-MU", "mg-MG", "mhr-RU", "mi-NZ", "miq-NI",
+  "mjw-IN", "mk-MK", "ml-IN", "mni-IN", "mn-MN", "mnw-MM", "mr-IN",
+  "ms-MY", "mt-MT", "my-MM", "nan-TW", "nb-NO", "nds-DE", "nds-NL",
+  "ne-NP", "nhn-MX", "niu-NU", "niu-NZ", "nl-AW", "nl-BE", "nl-NL",
+  "nn-NO", "nr-ZA", "nso-ZA", "oc-FR", "om-ET", "om-KE", "or-IN",
+  "os-RU", "pa-IN", "pap-AN", "pap-AW", "pap-CW", "pa-PK", "pl-PL",
+  "ps-AF", "pt-BR", "pt-PT", "quz-PE", "raj-IN", "ro-RO", "ru-RU",
+  "ru-UA", "rw-RW", "sah-RU", "sa-IN", "sat-IN", "sc-IT", "sd-IN",
+  "se-NO", "sgs-LT", "shn-MM", "shs-CA", "sid-ET", "si-LK", "sk-SK",
+  "sl-SI", "sm-WS", "so-DJ", "so-ET", "so-KE", "so-SO", "sq-AL",
+  "sq-XK", "sq-MK", "sr-ME", "sr-RS", "ss-ZA", "st-ZA", "sv-FI",
+  "sv-SE", "sw-KE", "sw-TZ", "szl-PL", "ta-IN", "ta-LK", "tcy-IN",
+  "te-IN", "tg-TJ", "the-NP", "th-TH", "ti-ER", "ti-ET", "tig-ER",
+  "tk-TM", "tn-ZA", "to-TO", "tpi-PG", "tr-CY", "tr-TR",
+  "ts-ZA", "tt-RU", "ug-CN", "uk-UA", "unm-US", "ur-IN", "ur-PK",
+  "uz-UZ", "ve-ZA", "vi-VN", "wa-BE", "wae-CH", "wal-ET", "wo-SN",
+  "xh-ZA", "yi-US", "yo-NG", "yue-HK", "yuw-PG", "zh-CN", "zh-HK",
+  "zh-SG", "zh-TW", "zu-ZA"
+]
+
+// Locales supported in the OS being used to run this script, which should be what's used for running Grist in production.
+const inNode = new Set(Intl.DisplayNames.supportedLocalesOf(localeCodes));
+
+// Locale supported in Python 2.7 (on Ubuntu 20.04).
+// Currently Python locales support is not that important, but might be in the future.
+const inPython = new Set([
+  "fr-LU", "li-BE", "ar-IN", "chr-US", "fr-CA", "cmn-TW", "hy-AM",
+  "lo-LA", "no-NO", "C", "fy-NL", "hr-HR", "cy-GB", "ik-CA",
+  "el-GR", "my-MM", "sat-IN", "kk-KZ", "agr-PE", "an-ES", "bn-BD",
+  "ta-LK", "so-ET", "mai-IN", "brx-IN", "nhn-MX", "ar-YE", "mjw-IN",
+  "es-BO", "zh-SG", "hu-HU", "br-FR", "ar-BH", "sl-CS", "hif-FJ",
+  "pp-AN", "pap-AN", "ur-PK", "bho-NP", "uz-UZ", "pl-PL", "se-NO",
+  "es-PE", "es-EC", "ug-CN", "zh-HK", "sd-IN", "ak-GH", "ja-JP",
+  "bg-BG", "en-IL", "eu-FR", "wal-ET", "km-KH", "ti-ER", "ar-SS",
+  "sm-WS", "yi-US", "hne-IN", "da-DK", "lzh-TW", "ve-ZA", "aa-ET",
+  "sq-MK", "es-SV", "lij-IT", "es-NI", "ta-IN", "sc-IT", "en-BE",
+  "sgs-LT", "raj-IN", "rw-RW", "tig-ER", "en-NZ", "sid-ET", "es-ES",
+  "crh-UA", "en-ZM", "ml-IN", "ks-IN", "bn-IN", "so-SO", "sv-FI",
+  "en-SC", "mag-IN", "nso-ZA", "ar-MA", "quz-PE", "sr-ME", "the-NP",
+  "ar-IQ", "fa-IR", "niu-NU", "mfe-MU", "tcy-IN", "de-LU", "az-IR",
+  "iw-IL", "szl-PL", "cs-CZ", "bhb-IN", "lb-LU", "en-AG", "de-BE",
+  "nds-DE", "iu-CA", "ku-TR", "dv-MV", "fy-DE", "lv-LV", "bem-ZM",
+  "mai-NP", "en-IE", "eo-US", "ts-ZA", "zh-TW", "bi-VU", "ar-JO",
+  "csb-PL", "tr-TR", "bs-BA", "es-UY", "niu-NZ", "en-CA", "kn-IN",
+  "en-ZS", "shs-CA", "tl-PH", "eu-ES", "el-CY", "sd-PK", "ber-DZ",
+  "ss-ZA", "eo", "az-AZ", "mr-IN", "nn-NO", "or-IN", "pd-DE",
+  "ht-HT", "ky-KG", "en-PH", "tt-RU", "am-ET", "gez-ET", "pap-CW",
+  "ph-PH", "id-ID", "ar-AA", "ce-RU", "ln-CD", "ko-KR", "gez-ER",
+  "sw-KE", "de-IT", "pa-PK", "fur-IT", "ar-KW", "si-LK", "ur-IN",
+  "unm-US", "fr-FR", "sr-RS", "pd-US", "ca-FR", "tg-TJ", "yuw-PG",
+  "ti-ET", "de-CH", "pt-PT", "en-SG", "ber-MA", "ig-NG", "gl-ES",
+  "ga-IE", "es-CO", "en-HK", "ar-SD", "af-ZA", "en-DL", "mi-NZ",
+  "eo-XX", "hak-TW", "kab-DZ", "ka-GE", "xh-ZA", "pa-IN", "en-DK",
+  "om-KE", "be-BY", "fr-BE", "th-TH", "wae-CH", "et-EE", "nl-NL",
+  "fil-PH", "es-GT", "mt-MT", "as-IN", "ps-AF", "yue-HK", "uk-UA",
+  "lt-LT", "ar-EG", "hi-IN", "om-ET", "ff-SN", "ar-LY", "tpi-PG",
+  "tk-TM", "tr-CY", "ar-LB", "es-CL", "ne-NP", "bo-CN", "es-MX",
+  "so-KE", "en-AU", "he-IL", "mni-IN", "sh-HR", "bo-IN", "sv-SE",
+  "gd-GB", "en-US", "ca-ES", "to-TO", "es-PR", "hsb-DE", "es-US",
+  "de-DE", "de-LI", "gv-GB", "st-ZA", "es-PA", "de-AT", "ru-RU",
+  "oc-FR", "fr-CH", "mk-MK", "ms-MY", "vi-VN", "es-CU", "nl-AW",
+  "aa-DJ", "ru-UA", "it-IT", "ny-NO", "en-IN", "ha-NG", "ar-OM",
+  "nds-NL", "aa-ER", "te-IN", "en-NG", "ast-ES", "nr-ZA", "mn-MN",
+  "mhr-RU", "is-IS", "fi-FI", "miq-NI", "ayc-PE", "byn-ER", "shn-MM",
+  "sl-SI", "ia-FR", "nb-NO", "es-VE", "kw-GB", "it-CH", "en-EN",
+  "pt-BR", "kok-IN", "ia", "sa-IN", "so-DJ", "es-AR", "en-BW",
+  "tn-ZA", "es-CR", "es-HN", "gu-IN", "ca-IT", "en-ZW", "nl-BE",
+  "dz-BT", "bho-IN", "kl-GL", "yo-NG", "li-NL", "wa-BE", "fo-FO",
+  "lg-UG", "sw-TZ", "ar-SA", "zu-ZA", "wo-SN", "ee-EE", "es-DO",
+  "ar-DZ", "mg-MG", "en-ZA", "ar-QA", "sk-SK", "ckb-IQ", "zh-CN",
+  "en-GB", "sr-CS", "ro-RO", "ar-TN", "ca-AD", "eo-EO", "cv-RU",
+  "es-PY", "sq-AL", "nan-TW", "doi-IN", "os-RU", "anp-IN", "pap-AW",
+  "ar-SY", "ar-AE"
+]);
+
+// Generate file content
+const isSupported = (locale) => [inNode, inPython].every(set => set.has(locale));
+const supportedList = localeCodes.filter(isSupported);
+// Convert to text, 7 codes per line
+const supportedText = supportedList
+  .map(locale => `"${locale}"`)
+  .reduce((list, locale) => {
+    let line = list.pop() || [];
+    if (line.length > 6) {
+      list.push(line);
+      line = []
+    }
+    line.push(locale);
+    list.push(line);
+    return list;
+  }, [])
+  .map(line => "  " + line.join(", "))
+  .join(",\n");
+
+const fileContent = `// This file was generated by core/buildtools/generate_locale_list.js at ${new Date().toISOString()}
+export const localeCodes = [
+${supportedText}
+];
+`
+const fs = require("fs");
+const path = require("path");
+fs.writeFileSync(path.join(__dirname, "../app/common/LocaleCodes.ts"), fileContent);