Merge pull request #240 from ronnix/ignore-diacritics-in-autocomplete

Ignore diacritics in autocomplete

app/client/lib/ACIndex.ts

@@ -10,6 +10,7 @@
 import {localeCompare, nativeCompare, sortedIndex} from 'app/common/gutil';
 import {DomContents} from 'grainjs';
 import escapeRegExp = require("lodash/escapeRegExp");
+import deburr = require("lodash/deburr");
 
 export interface ACItem {
   // This should be a trimmed lowercase version of the item's text. It may be an accessor.
@@ -17,6 +18,13 @@ export interface ACItem {
   cleanText: string;
 }
 
+// Returns a trimmed, lowercase version of a string,
+// from which accents and other diacritics have been removed,
+// so that autocomplete is case- and accent-insensitive.
+export function normalizeText(text: string): string {
+  return deburr(text).trim().toLowerCase();
+}
+
 // Regexp used to split text into words; includes nearly all punctuation. This means that
 // "foo-bar" may be searched by "bar", but it's impossible to search for punctuation itself (e.g.
 // "a-b" and "a+b" are not distinguished). (It's easy to exclude unicode punctuation too if the
@@ -91,7 +99,7 @@ export class ACIndexImpl<Item extends ACItem> implements ACIndex<Item> {
   // The main search function. SearchText will be cleaned (trimmed and lowercased) at the start.
   // Empty search text returns the first N items in the search universe.
   public search(searchText: string): ACResults<Item> {
-    const cleanedSearchText = searchText.trim().toLowerCase();
+    const cleanedSearchText = normalizeText(searchText);
     const searchWords = cleanedSearchText.split(wordSepRegexp).filter(w => w);
 
     // Maps item index in _allItems to its score.

app/client/models/ColumnACIndexes.ts

@@ -8,7 +8,7 @@
  *
  * It is currently used for auto-complete in the ReferenceEditor and ReferenceListEditor widgets.
  */
-import {ACIndex, ACIndexImpl} from 'app/client/lib/ACIndex';
+import {ACIndex, ACIndexImpl, normalizeText} from 'app/client/lib/ACIndex';
 import {ColumnCache} from 'app/client/models/ColumnCache';
 import {UserError} from 'app/client/models/errors';
 import {TableData} from 'app/client/models/TableData';
@@ -45,7 +45,7 @@ export class ColumnACIndexes {
     const items: ICellItem[] = valColumn.map((val, i) => {
       const rowId = rowIds[i];
       const text = formatter.formatAny(val);
-      const cleanText = text.trim().toLowerCase();
+      const cleanText = normalizeText(text);
       return {rowId, text, cleanText};
     });
     items.sort(itemCompare);

app/client/widgets/ChoiceListEditor.ts

@@ -1,5 +1,5 @@
 import {createGroup} from 'app/client/components/commands';
-import {ACIndexImpl, ACItem, ACResults, buildHighlightedDom, HighlightFunc} from 'app/client/lib/ACIndex';
+import {ACIndexImpl, ACItem, ACResults, buildHighlightedDom, normalizeText, HighlightFunc} from 'app/client/lib/ACIndex';
 import {IAutocompleteOptions} from 'app/client/lib/autocomplete';
 import {IToken, TokenField, tokenFieldStyles} from 'app/client/lib/TokenField';
 import {colors, testId} from 'app/client/ui2018/cssVars';
@@ -16,7 +16,7 @@ import {choiceToken, cssChoiceACItem, cssChoiceToken} from 'app/client/widgets/C
 import {icon} from 'app/client/ui2018/icons';
 
 export class ChoiceItem implements ACItem, IToken {
-  public cleanText: string = this.label.toLowerCase().trim();
+  public cleanText: string = normalizeText(this.label);
   constructor(
     public label: string,
     public isInvalid: boolean,  // If set, this token is not one of the valid choices.

app/client/widgets/ReferenceEditor.ts

@@ -1,4 +1,4 @@
-import { ACResults, buildHighlightedDom, HighlightFunc } from 'app/client/lib/ACIndex';
+import { ACResults, buildHighlightedDom, normalizeText, HighlightFunc } from 'app/client/lib/ACIndex';
 import { Autocomplete } from 'app/client/lib/autocomplete';
 import { ICellItem } from 'app/client/models/ColumnACIndexes';
 import { reportError } from 'app/client/models/errors';
@@ -115,7 +115,7 @@ export class ReferenceEditor extends NTextEditor {
     this._showAddNew = false;
     if (!this._enableAddNew || !text) { return result; }
 
-    const cleanText = text.trim().toLowerCase();
+    const cleanText = normalizeText(text);
     if (result.items.find((item) => item.cleanText === cleanText)) {
       return result;
     }

app/client/widgets/ReferenceListEditor.ts

@@ -1,5 +1,5 @@
 import { createGroup } from 'app/client/components/commands';
-import { ACItem, ACResults, HighlightFunc } from 'app/client/lib/ACIndex';
+import { ACItem, ACResults, normalizeText, HighlightFunc } from 'app/client/lib/ACIndex';
 import { IAutocompleteOptions } from 'app/client/lib/autocomplete';
 import { IToken, TokenField, tokenFieldStyles } from 'app/client/lib/TokenField';
 import { reportError } from 'app/client/models/errors';
@@ -26,7 +26,7 @@ class ReferenceItem implements IToken, ACItem {
    * similar to getItemText() from IAutocompleteOptions.
    */
   public label: string = typeof this.rowId === 'number' ? String(this.rowId) : this.text;
-  public cleanText: string = this.text.trim().toLowerCase();
+  public cleanText: string = normalizeText(this.text);
 
   constructor(
     public text: string,
@@ -264,7 +264,7 @@ export class ReferenceListEditor extends NewBaseEditor {
     this._showAddNew = false;
     if (!this._enableAddNew || !text) { return result; }
 
-    const cleanText = text.trim().toLowerCase();
+    const cleanText = normalizeText(text);
     if (result.items.find((item) => item.cleanText === cleanText)) {
       return result;
     }

test/nbrowser/ChoiceList.ts

@@ -301,6 +301,23 @@ describe('ChoiceList', function() {
     await gu.waitForServer();
     assert.equal(await driver.find('.cell_editor').isPresent(), false);
     assert.equal(await gu.getCell({rowNum: 1, col: 'B'}).getText(), 'Blue\nGreen\nBlack');
+
+
+    // Starting to type names without accents should match the actual choices
+    await gu.addColumn("Accents");
+    await api.applyUserActions(docId, [
+      ['ModifyColumn', 'Table1', 'Accents', {
+        type: 'ChoiceList',
+        widgetOptions: JSON.stringify({
+          choices: ['Adélaïde', 'Adèle', 'Agnès', 'Amélie'],
+        })
+      }],
+    ]);
+    await gu.getCell({rowNum: 1, col: 'Accents'}).click();
+    await driver.sendKeys('Ade', Key.ENTER);
+    await driver.sendKeys('Agne', Key.ENTER);
+    await driver.sendKeys('Ame', Key.ENTER);
+    assert.deepEqual(await getEditorTokens(), ['Adélaïde', 'Agnès', 'Amélie']);
   });
 
   it('should be visible in formulas', async () => {

test/nbrowser/ReferenceColumns.ts

@@ -413,6 +413,18 @@ describe('ReferenceColumns', function() {
         ['Dark Slate Blue', 'Dark Slate Gray', 'Slate Blue', 'Medium Slate Blue']);
       await driver.sendKeys(Key.ESCAPE);
 
+      // Starting to type Añil with the accent
+      await driver.sendKeys('añ');
+      assert.deepEqual(await getACOptions(2),
+        ['Añil', 'Alice Blue']);
+      await driver.sendKeys(Key.ESCAPE);
+
+      // Starting to type Añil without the accent should work too
+      await driver.sendKeys('an');
+      assert.deepEqual(await getACOptions(2),
+        ['Añil', 'Alice Blue']);
+      await driver.sendKeys(Key.ESCAPE);
+
       await driver.sendKeys('blac');
       assert.deepEqual(await getACOptions(6),
         ['Black', 'Blanched Almond', 'Blue', 'Blue Violet', 'Alice Blue', 'Cadet Blue']);

test/nbrowser/ReferenceList.ts

@@ -709,6 +709,18 @@ describe('ReferenceList', function() {
         ['Dark Slate Blue', 'Dark Slate Gray', 'Slate Blue', 'Medium Slate Blue']);
       await driver.sendKeys(Key.ESCAPE);
 
+      // Starting to type Añil with the accent
+      await driver.sendKeys('añ');
+      assert.deepEqual(await getACOptions(2),
+        ['Añil', 'Alice Blue']);
+      await driver.sendKeys(Key.ESCAPE);
+
+      // Starting to type Añil without the accent should work too
+      await driver.sendKeys('an');
+      assert.deepEqual(await getACOptions(2),
+        ['Añil', 'Alice Blue']);
+      await driver.sendKeys(Key.ESCAPE);
+
       await driver.sendKeys('blac');
       assert.deepEqual(await getACOptions(6),
         ['Black', 'Blanched Almond', 'Blue', 'Blue Violet', 'Alice Blue', 'Cadet Blue']);