Ignore diacritics in autocomplete

Works for:
- Choice
- Choice List
- Reference
- Reference List

Co-Authored-By: Louis Delbosc <louis.delbosc.prestataire@anct.gouv.fr>

app/client/lib/ACIndex.ts

@@ -7,7 +7,7 @@
  * "lush" would only match the "L" in "Lavender".
  */
 
-import {localeCompare, nativeCompare, sortedIndex} from 'app/common/gutil';
+import {localeCompare, nativeCompare, removeDiacritics, sortedIndex} from 'app/common/gutil';
 import {DomContents} from 'grainjs';
 import escapeRegExp = require("lodash/escapeRegExp");
 
@@ -17,6 +17,12 @@ export interface ACItem {
   cleanText: string;
 }
 
+// Returns a normalized, trimmed, lowercase version of a string, so that autocomplete is case-
+// and accent-insensitive.
+export function cleanText(text: string): string {
+  return removeDiacritics(text).trim().toLowerCase();
+}
+
 // Regexp used to split text into words; includes nearly all punctuation. This means that
 // "foo-bar" may be searched by "bar", but it's impossible to search for punctuation itself (e.g.
 // "a-b" and "a+b" are not distinguished). (It's easy to exclude unicode punctuation too if the
@@ -91,7 +97,7 @@ export class ACIndexImpl<Item extends ACItem> implements ACIndex<Item> {
   // The main search function. SearchText will be cleaned (trimmed and lowercased) at the start.
   // Empty search text returns the first N items in the search universe.
   public search(searchText: string): ACResults<Item> {
-    const cleanedSearchText = searchText.trim().toLowerCase();
+    const cleanedSearchText = cleanText(searchText);
     const searchWords = cleanedSearchText.split(wordSepRegexp).filter(w => w);
 
     // Maps item index in _allItems to its score.

app/client/models/ColumnACIndexes.ts

@@ -8,7 +8,7 @@
  *
  * It is currently used for auto-complete in the ReferenceEditor and ReferenceListEditor widgets.
  */
-import {ACIndex, ACIndexImpl} from 'app/client/lib/ACIndex';
+import {ACIndex, ACIndexImpl, cleanText as clean} from 'app/client/lib/ACIndex';
 import {ColumnCache} from 'app/client/models/ColumnCache';
 import {UserError} from 'app/client/models/errors';
 import {TableData} from 'app/client/models/TableData';
@@ -45,7 +45,7 @@ export class ColumnACIndexes {
     const items: ICellItem[] = valColumn.map((val, i) => {
       const rowId = rowIds[i];
       const text = formatter.formatAny(val);
-      const cleanText = text.trim().toLowerCase();
+      const cleanText = clean(text);
       return {rowId, text, cleanText};
     });
     items.sort(itemCompare);

app/client/widgets/ChoiceListEditor.ts

@@ -1,5 +1,5 @@
 import {createGroup} from 'app/client/components/commands';
-import {ACIndexImpl, ACItem, ACResults, buildHighlightedDom, HighlightFunc} from 'app/client/lib/ACIndex';
+import {ACIndexImpl, ACItem, ACResults, buildHighlightedDom, cleanText as clean, HighlightFunc} from 'app/client/lib/ACIndex';
 import {IAutocompleteOptions} from 'app/client/lib/autocomplete';
 import {IToken, TokenField, tokenFieldStyles} from 'app/client/lib/TokenField';
 import {colors, testId} from 'app/client/ui2018/cssVars';
@@ -16,7 +16,7 @@ import {choiceToken, cssChoiceACItem, cssChoiceToken} from 'app/client/widgets/C
 import {icon} from 'app/client/ui2018/icons';
 
 export class ChoiceItem implements ACItem, IToken {
-  public cleanText: string = this.label.toLowerCase().trim();
+  public cleanText: string = clean(this.label);
   constructor(
     public label: string,
     public isInvalid: boolean,  // If set, this token is not one of the valid choices.

app/client/widgets/ReferenceEditor.ts

@@ -1,4 +1,4 @@
-import { ACResults, buildHighlightedDom, HighlightFunc } from 'app/client/lib/ACIndex';
+import { ACResults, buildHighlightedDom, cleanText as clean, HighlightFunc } from 'app/client/lib/ACIndex';
 import { Autocomplete } from 'app/client/lib/autocomplete';
 import { ICellItem } from 'app/client/models/ColumnACIndexes';
 import { reportError } from 'app/client/models/errors';
@@ -115,7 +115,7 @@ export class ReferenceEditor extends NTextEditor {
     this._showAddNew = false;
     if (!this._enableAddNew || !text) { return result; }
 
-    const cleanText = text.trim().toLowerCase();
+    const cleanText = clean(text);
     if (result.items.find((item) => item.cleanText === cleanText)) {
       return result;
     }

app/client/widgets/ReferenceListEditor.ts

@@ -1,5 +1,5 @@
 import { createGroup } from 'app/client/components/commands';
-import { ACItem, ACResults, HighlightFunc } from 'app/client/lib/ACIndex';
+import { ACItem, ACResults, cleanText as clean, HighlightFunc } from 'app/client/lib/ACIndex';
 import { IAutocompleteOptions } from 'app/client/lib/autocomplete';
 import { IToken, TokenField, tokenFieldStyles } from 'app/client/lib/TokenField';
 import { reportError } from 'app/client/models/errors';
@@ -26,7 +26,7 @@ class ReferenceItem implements IToken, ACItem {
    * similar to getItemText() from IAutocompleteOptions.
    */
   public label: string = typeof this.rowId === 'number' ? String(this.rowId) : this.text;
-  public cleanText: string = this.text.trim().toLowerCase();
+  public cleanText: string = clean(this.text);
 
   constructor(
     public text: string,
@@ -264,7 +264,7 @@ export class ReferenceListEditor extends NewBaseEditor {
     this._showAddNew = false;
     if (!this._enableAddNew || !text) { return result; }
 
-    const cleanText = text.trim().toLowerCase();
+    const cleanText = clean(text);
     if (result.items.find((item) => item.cleanText === cleanText)) {
       return result;
     }

app/common/gutil.ts

@@ -55,6 +55,11 @@ export function capitalizeFirstWord(str: string): string {
   return str.replace(/\b[a-z]/i, c => c.toUpperCase());
 }
 
+// Remove diacritics (accents and other signs).
+export function removeDiacritics(text: string): string {
+  return text.normalize("NFKD").replace(/[\u0300-\u036f]/g, "")
+}
+
 // Returns whether the string n represents a valid number.
 // http://stackoverflow.com/questions/18082/validate-numbers-in-javascript-isnumeric
 export function isNumber(n: string): boolean {