(core) updates from grist-core

README.md

@@ -98,6 +98,10 @@ To enable gVisor sandboxing, set `--env GRIST_SANDBOX_FLAVOR=gvisor`.
 This should work with default docker settings, but may not work in all
 environments.
 
+You can find a lot more about configuring Grist, setting up authentication,
+and running it on a public server in our
+[Self-Managed Grist](https://support.getgrist.com/self-managed/) handbook.
+
 ## Building from source
 
 To build Grist from source, follow these steps:
@@ -137,11 +141,10 @@ You can change your name in `Profile Settings` in
 the [User Menu](https://support.getgrist.com/glossary/#user-menu).
 
 For multi-user operation, or if you wish to access Grist across the
-public internet, you'll want to connect it to your own single sign-in service.
+public internet, you'll want to connect it to your own Single Sign-On service.
-There's a `docker-compose` template at https://community.getgrist.com/t/a-template-for-self-hosting-grist-with-traefik-and-docker-compose/856
+There are a lot of ways to do this, including [SAML and forward authentication](https://support.getgrist.com/self-managed/#how-do-i-set-up-authentication).
-covering using Let's Encrypt for certificates and Google for sign-ins.
+Grist has been tested with [Authentik](https://goauthentik.io/), [Auth0](https://auth0.com/),
-You can also use [SAML](https://github.com/gristlabs/grist-core/blob/main/app/server/lib/SamlConfig.ts).
+and Google/Microsoft sign-ins via [Dex](https://dexidp.io/).
-Grist has been tested with [Authentik](https://goauthentik.io/) and [Auth0](https://auth0.com/).
 
 ## Why free and open source software
 
@@ -151,7 +154,7 @@ here, combined with business-specific software designed to scale it to many user
 etc.
 
 Grist Labs is an open-core company. We offer Grist hosting as a
-service, with free and paid plans. We intend to also develop and sell
+service, with free and paid plans. We also develop and sell
 features related to Grist using a proprietary license, targeted at the
 needs of enterprises with large self-managed installations. We see
 data portability and autonomy as a key value Grist can bring to our

app/client/lib/ACIndex.ts

@@ -10,6 +10,7 @@
 import {localeCompare, nativeCompare, sortedIndex} from 'app/common/gutil';
 import {DomContents} from 'grainjs';
 import escapeRegExp = require("lodash/escapeRegExp");
+import deburr = require("lodash/deburr");
 
 export interface ACItem {
   // This should be a trimmed lowercase version of the item's text. It may be an accessor.
@@ -17,6 +18,13 @@ export interface ACItem {
   cleanText: string;
 }
 
+// Returns a trimmed, lowercase version of a string,
+// from which accents and other diacritics have been removed,
+// so that autocomplete is case- and accent-insensitive.
+export function normalizeText(text: string): string {
+  return deburr(text).trim().toLowerCase();
+}
+
 // Regexp used to split text into words; includes nearly all punctuation. This means that
 // "foo-bar" may be searched by "bar", but it's impossible to search for punctuation itself (e.g.
 // "a-b" and "a+b" are not distinguished). (It's easy to exclude unicode punctuation too if the
@@ -91,7 +99,7 @@ export class ACIndexImpl<Item extends ACItem> implements ACIndex<Item> {
   // The main search function. SearchText will be cleaned (trimmed and lowercased) at the start.
   // Empty search text returns the first N items in the search universe.
   public search(searchText: string): ACResults<Item> {
-    const cleanedSearchText = searchText.trim().toLowerCase();
+    const cleanedSearchText = normalizeText(searchText);
     const searchWords = cleanedSearchText.split(wordSepRegexp).filter(w => w);
 
     // Maps item index in _allItems to its score.

app/client/models/ColumnACIndexes.ts

@@ -8,7 +8,7 @@
  *
  * It is currently used for auto-complete in the ReferenceEditor and ReferenceListEditor widgets.
  */
-import {ACIndex, ACIndexImpl} from 'app/client/lib/ACIndex';
+import {ACIndex, ACIndexImpl, normalizeText} from 'app/client/lib/ACIndex';
 import {ColumnCache} from 'app/client/models/ColumnCache';
 import {UserError} from 'app/client/models/errors';
 import {TableData} from 'app/client/models/TableData';
@@ -45,7 +45,7 @@ export class ColumnACIndexes {
     const items: ICellItem[] = valColumn.map((val, i) => {
       const rowId = rowIds[i];
       const text = formatter.formatAny(val);
-      const cleanText = text.trim().toLowerCase();
+      const cleanText = normalizeText(text);
       return {rowId, text, cleanText};
     });
     items.sort(itemCompare);

app/client/widgets/ChoiceListEditor.ts

@@ -1,5 +1,5 @@
 import {createGroup} from 'app/client/components/commands';
-import {ACIndexImpl, ACItem, ACResults, buildHighlightedDom, HighlightFunc} from 'app/client/lib/ACIndex';
+import {ACIndexImpl, ACItem, ACResults, buildHighlightedDom, normalizeText, HighlightFunc} from 'app/client/lib/ACIndex';
 import {IAutocompleteOptions} from 'app/client/lib/autocomplete';
 import {IToken, TokenField, tokenFieldStyles} from 'app/client/lib/TokenField';
 import {colors, testId} from 'app/client/ui2018/cssVars';
@@ -16,7 +16,7 @@ import {choiceToken, cssChoiceACItem, cssChoiceToken} from 'app/client/widgets/C
 import {icon} from 'app/client/ui2018/icons';
 
 export class ChoiceItem implements ACItem, IToken {
-  public cleanText: string = this.label.toLowerCase().trim();
+  public cleanText: string = normalizeText(this.label);
   constructor(
     public label: string,
     public isInvalid: boolean,  // If set, this token is not one of the valid choices.

app/client/widgets/ReferenceEditor.ts

@@ -1,4 +1,4 @@
-import { ACResults, buildHighlightedDom, HighlightFunc } from 'app/client/lib/ACIndex';
+import { ACResults, buildHighlightedDom, normalizeText, HighlightFunc } from 'app/client/lib/ACIndex';
 import { Autocomplete } from 'app/client/lib/autocomplete';
 import { ICellItem } from 'app/client/models/ColumnACIndexes';
 import { reportError } from 'app/client/models/errors';
@@ -115,7 +115,7 @@ export class ReferenceEditor extends NTextEditor {
     this._showAddNew = false;
     if (!this._enableAddNew || !text) { return result; }
 
-    const cleanText = text.trim().toLowerCase();
+    const cleanText = normalizeText(text);
     if (result.items.find((item) => item.cleanText === cleanText)) {
       return result;
     }

app/client/widgets/ReferenceListEditor.ts

@@ -1,5 +1,5 @@
 import { createGroup } from 'app/client/components/commands';
-import { ACItem, ACResults, HighlightFunc } from 'app/client/lib/ACIndex';
+import { ACItem, ACResults, normalizeText, HighlightFunc } from 'app/client/lib/ACIndex';
 import { IAutocompleteOptions } from 'app/client/lib/autocomplete';
 import { IToken, TokenField, tokenFieldStyles } from 'app/client/lib/TokenField';
 import { reportError } from 'app/client/models/errors';
@@ -26,7 +26,7 @@ class ReferenceItem implements IToken, ACItem {
    * similar to getItemText() from IAutocompleteOptions.
    */
   public label: string = typeof this.rowId === 'number' ? String(this.rowId) : this.text;
-  public cleanText: string = this.text.trim().toLowerCase();
+  public cleanText: string = normalizeText(this.text);
 
   constructor(
     public text: string,
@@ -264,7 +264,7 @@ export class ReferenceListEditor extends NewBaseEditor {
     this._showAddNew = false;
     if (!this._enableAddNew || !text) { return result; }
 
-    const cleanText = text.trim().toLowerCase();
+    const cleanText = normalizeText(text);
     if (result.items.find((item) => item.cleanText === cleanText)) {
       return result;
     }

test/nbrowser/ChoiceList.ts

@@ -301,6 +301,23 @@ describe('ChoiceList', function() {
     await gu.waitForServer();
     assert.equal(await driver.find('.cell_editor').isPresent(), false);
     assert.equal(await gu.getCell({rowNum: 1, col: 'B'}).getText(), 'Blue\nGreen\nBlack');
+
+
+    // Starting to type names without accents should match the actual choices
+    await gu.addColumn("Accents");
+    await api.applyUserActions(docId, [
+      ['ModifyColumn', 'Table1', 'Accents', {
+        type: 'ChoiceList',
+        widgetOptions: JSON.stringify({
+          choices: ['Adélaïde', 'Adèle', 'Agnès', 'Amélie'],
+        })
+      }],
+    ]);
+    await gu.getCell({rowNum: 1, col: 'Accents'}).click();
+    await driver.sendKeys('Ade', Key.ENTER);
+    await driver.sendKeys('Agne', Key.ENTER);
+    await driver.sendKeys('Ame', Key.ENTER);
+    assert.deepEqual(await getEditorTokens(), ['Adélaïde', 'Agnès', 'Amélie']);
   });
 
   it('should be visible in formulas', async () => {

test/nbrowser/ReferenceColumns.ts

@@ -413,6 +413,18 @@ describe('ReferenceColumns', function() {
         ['Dark Slate Blue', 'Dark Slate Gray', 'Slate Blue', 'Medium Slate Blue']);
       await driver.sendKeys(Key.ESCAPE);
 
+      // Starting to type Añil with the accent
+      await driver.sendKeys('añ');
+      assert.deepEqual(await getACOptions(2),
+        ['Añil', 'Alice Blue']);
+      await driver.sendKeys(Key.ESCAPE);
+
+      // Starting to type Añil without the accent should work too
+      await driver.sendKeys('an');
+      assert.deepEqual(await getACOptions(2),
+        ['Añil', 'Alice Blue']);
+      await driver.sendKeys(Key.ESCAPE);
+
       await driver.sendKeys('blac');
       assert.deepEqual(await getACOptions(6),
         ['Black', 'Blanched Almond', 'Blue', 'Blue Violet', 'Alice Blue', 'Cadet Blue']);

test/nbrowser/ReferenceList.ts

@@ -710,6 +710,18 @@ describe('ReferenceList', function() {
         ['Dark Slate Blue', 'Dark Slate Gray', 'Slate Blue', 'Medium Slate Blue']);
       await driver.sendKeys(Key.ESCAPE);
 
+      // Starting to type Añil with the accent
+      await driver.sendKeys('añ');
+      assert.deepEqual(await getACOptions(2),
+        ['Añil', 'Alice Blue']);
+      await driver.sendKeys(Key.ESCAPE);
+
+      // Starting to type Añil without the accent should work too
+      await driver.sendKeys('an');
+      assert.deepEqual(await getACOptions(2),
+        ['Añil', 'Alice Blue']);
+      await driver.sendKeys(Key.ESCAPE);
+
       await driver.sendKeys('blac');
       assert.deepEqual(await getACOptions(6),
         ['Black', 'Blanched Almond', 'Blue', 'Blue Violet', 'Alice Blue', 'Cadet Blue']);