(core) Add dropdown conditions

Summary:
Dropdown conditions let you specify a predicate formula that's used to filter
choices and references in their respective autocomplete dropdown menus.

Test Plan: Python and browser tests (WIP).

Reviewers: jarek, paulfitz

Reviewed By: jarek

Subscribers: dsagal, paulfitz

Differential Revision: https://phab.getgrist.com/D4235

app/client/lib/ACIndex.ts

@@ -51,6 +51,9 @@ export interface ACResults<Item extends ACItem> {
   // Matching items in order from best match to worst.
   items: Item[];
 
+  // Additional items to show (e.g. the "Add New" item, for Choice and Reference fields).
+  extraItems: Item[];
+
   // May be used to highlight matches using buildHighlightedDom().
   highlightFunc: HighlightFunc;
 
@@ -159,7 +162,7 @@ export class ACIndexImpl<Item extends ACItem> implements ACIndex<Item> {
 
     if (!cleanedSearchText) {
       // In this case we are just returning the first few items.
-      return {items, highlightFunc: highlightNone, selectIndex: -1};
+      return {items, extraItems: [], highlightFunc: highlightNone, selectIndex: -1};
     }
 
     const highlightFunc = highlightMatches.bind(null, searchWords);
@@ -170,7 +173,7 @@ export class ACIndexImpl<Item extends ACItem> implements ACIndex<Item> {
     if (selectIndex >= 0 && !startsWithText(items[selectIndex], cleanedSearchText, searchWords)) {
       selectIndex = -1;
     }
-    return {items, highlightFunc, selectIndex};
+    return {items, extraItems: [], highlightFunc, selectIndex};
   }
 
   /**

app/client/lib/ACUserManager.ts

@@ -98,7 +98,7 @@ export function buildACMemberEmail(
         label: text,
         id: 0,
       };
-      results.items.push(newObject);
+      results.extraItems.push(newObject);
     }
     return results;
   };

app/client/lib/DocPluginManager.ts

@@ -4,8 +4,6 @@ import {SafeBrowser} from 'app/client/lib/SafeBrowser';
 import {ActiveDocAPI} from 'app/common/ActiveDocAPI';
 import {LocalPlugin} from 'app/common/plugin';
 import {createRpcLogger, PluginInstance} from 'app/common/PluginInstance';
-import {Theme} from 'app/common/ThemePrefs';
-import {Computed} from 'grainjs';
 import {Rpc} from 'grain-rpc';
 
 /**
@@ -18,7 +16,6 @@ export class DocPluginManager {
   private _clientScope = this._options.clientScope;
   private _docComm = this._options.docComm;
   private _localPlugins = this._options.plugins;
-  private _theme = this._options.theme;
   private _untrustedContentOrigin = this._options.untrustedContentOrigin;
 
   constructor(private _options: {
@@ -26,7 +23,6 @@ export class DocPluginManager {
     untrustedContentOrigin: string,
     docComm: ActiveDocAPI,
     clientScope: ClientScope,
-    theme: Computed<Theme>,
   }) {
     this.pluginsList = [];
     for (const plugin of this._localPlugins) {
@@ -38,7 +34,6 @@ export class DocPluginManager {
           clientScope: this._clientScope,
           untrustedContentOrigin: this._untrustedContentOrigin,
           mainPath: components.safeBrowser,
-          theme: this._theme,
         });
         if (components.safeBrowser) {
           pluginInstance.rpc.registerForwarder(components.safeBrowser, safeBrowser);

app/client/lib/HomePluginManager.ts

@@ -2,8 +2,6 @@ import {ClientScope} from 'app/client/components/ClientScope';
 import {SafeBrowser} from 'app/client/lib/SafeBrowser';
 import {LocalPlugin} from 'app/common/plugin';
 import {createRpcLogger, PluginInstance} from 'app/common/PluginInstance';
-import {Theme} from 'app/common/ThemePrefs';
-import {Computed} from 'grainjs';
 
 /**
  * Home plugins are all plugins that contributes to a general Grist management tasks.
@@ -19,9 +17,8 @@ export class HomePluginManager {
     localPlugins: LocalPlugin[],
     untrustedContentOrigin: string,
     clientScope: ClientScope,
-    theme: Computed<Theme>,
   }) {
-    const {localPlugins, untrustedContentOrigin, clientScope, theme} = options;
+    const {localPlugins, untrustedContentOrigin, clientScope} = options;
     this.pluginsList = [];
     for (const plugin of localPlugins) {
       try {
@@ -41,7 +38,6 @@ export class HomePluginManager {
           clientScope,
           untrustedContentOrigin,
           mainPath: components.safeBrowser,
-          theme,
         });
         if (components.safeBrowser) {
           pluginInstance.rpc.registerForwarder(components.safeBrowser, safeBrowser);

app/client/lib/ReferenceUtils.ts

@@ -1,22 +1,36 @@
+import {ACIndex, ACResults} from 'app/client/lib/ACIndex';
+import {makeT} from 'app/client/lib/localization';
+import {ICellItem} from 'app/client/models/ColumnACIndexes';
+import {ColumnCache} from 'app/client/models/ColumnCache';
 import {DocData} from 'app/client/models/DocData';
 import {ColumnRec} from 'app/client/models/entities/ColumnRec';
 import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec';
 import {TableData} from 'app/client/models/TableData';
 import {getReferencedTableId, isRefListType} from 'app/common/gristTypes';
+import {EmptyRecordView} from 'app/common/PredicateFormula';
 import {BaseFormatter} from 'app/common/ValueFormatter';
+import {Disposable, dom, Observable} from 'grainjs';
+
+const t = makeT('ReferenceUtils');
 
 /**
  * Utilities for common operations involving Ref[List] fields.
  */
-export class ReferenceUtils {
+export class ReferenceUtils extends Disposable {
   public readonly refTableId: string;
   public readonly tableData: TableData;
   public readonly visibleColFormatter: BaseFormatter;
   public readonly visibleColModel: ColumnRec;
   public readonly visibleColId: string;
   public readonly isRefList: boolean;
+  public readonly hasDropdownCondition = Boolean(this.field.dropdownCondition.peek()?.text);
+
+  private readonly _columnCache: ColumnCache<ACIndex<ICellItem>>;
+  private _dropdownConditionError = Observable.create<string | null>(this, null);
+
+  constructor(public readonly field: ViewFieldRec, private readonly _docData: DocData) {
+    super();
 
-  constructor(public readonly field: ViewFieldRec, docData: DocData) {
     const colType = field.column().type();
     const refTableId = getReferencedTableId(colType);
     if (!refTableId) {
@@ -24,7 +38,7 @@ export class ReferenceUtils {
     }
     this.refTableId = refTableId;
 
-    const tableData = docData.getTable(refTableId);
+    const tableData = _docData.getTable(refTableId);
     if (!tableData) {
       throw new Error("Invalid referenced table " + refTableId);
     }
@@ -34,6 +48,8 @@ export class ReferenceUtils {
     this.visibleColModel = field.visibleColModel();
     this.visibleColId = this.visibleColModel.colId() || 'id';
     this.isRefList = isRefListType(colType);
+
+    this._columnCache = new ColumnCache<ACIndex<ICellItem>>(this.tableData);
   }
 
   public idToText(value: unknown) {
@@ -43,10 +59,86 @@ export class ReferenceUtils {
     return String(value || '');
   }
 
-  public autocompleteSearch(text: string) {
-    const acIndex = this.tableData.columnACIndexes.getColACIndex(this.visibleColId, this.visibleColFormatter);
+  /**
+   * Searches the autocomplete index for the given `text`, returning
+   * all matching results and related metadata.
+   *
+   * If a dropdown condition is set, results are dependent on the `rowId`
+   * that the autocomplete dropdown is open in. Otherwise, `rowId` has no
+   * effect.
+   */
+  public autocompleteSearch(text: string, rowId: number): ACResults<ICellItem> {
+    let acIndex: ACIndex<ICellItem>;
+    if (this.hasDropdownCondition) {
+      try {
+        acIndex = this._getDropdownConditionACIndex(rowId);
+      } catch (e) {
+        this._dropdownConditionError?.set(e);
+        return {items: [], extraItems: [], highlightFunc: () => [], selectIndex: -1};
+      }
+    } else {
+      acIndex = this.tableData.columnACIndexes.getColACIndex(
+        this.visibleColId,
+        this.visibleColFormatter,
+      );
+    }
     return acIndex.search(text);
   }
+
+  public buildNoItemsMessage() {
+    return dom.domComputed(use => {
+      const error = use(this._dropdownConditionError);
+      if (error) { return t('Error in dropdown condition'); }
+
+      return this.hasDropdownCondition
+        ? t('No choices matching condition')
+        : t('No choices to select');
+    });
+  }
+
+  /**
+   * Returns a column index for the visible column, filtering the items in the
+   * index according to the set dropdown condition.
+   *
+   * This method is similar to `this.tableData.columnACIndexes.getColACIndex`,
+   * but whereas that method caches indexes globally, this method does so
+   * locally (as a new instances of this class is created each time a Reference
+   * or Reference List editor is created).
+   *
+   * It's important that this method be used when a dropdown condition is set,
+   * as items in indexes that don't satisfy the dropdown condition need to be
+   * filtered.
+   */
+  private _getDropdownConditionACIndex(rowId: number) {
+    return this._columnCache.getValue(
+      this.visibleColId,
+      () => this.tableData.columnACIndexes.buildColACIndex(
+        this.visibleColId,
+        this.visibleColFormatter,
+        this._buildDropdownConditionACFilter(rowId)
+      )
+    );
+  }
+
+  private _buildDropdownConditionACFilter(rowId: number) {
+    const dropdownConditionCompiled = this.field.dropdownConditionCompiled.get();
+    if (dropdownConditionCompiled?.kind !== 'success') {
+      throw new Error('Dropdown condition is not compiled');
+    }
+
+    const tableId = this.field.tableId.peek();
+    const table = this._docData.getTable(tableId);
+    if (!table) { throw new Error(`Table ${tableId} not found`); }
+
+    const {result: predicate} = dropdownConditionCompiled;
+    const rec = table.getRecord(rowId) || new EmptyRecordView();
+    return (item: ICellItem) => {
+      const choice = item.rowId === 'new' ? new EmptyRecordView() : this.tableData.getRecord(item.rowId);
+      if (!choice) { throw new Error(`Reference ${item.rowId} not found`); }
+
+      return predicate({rec, choice});
+    };
+  }
 }
 
 export function nocaseEqual(a: string, b: string) {

app/client/lib/SafeBrowser.ts

@@ -33,6 +33,7 @@ import { ClientScope } from 'app/client/components/ClientScope';
 import { get as getBrowserGlobals } from 'app/client/lib/browserGlobals';
 import dom from 'app/client/lib/dom';
 import * as Mousetrap from 'app/client/lib/Mousetrap';
+import { gristThemeObs } from 'app/client/ui2018/theme';
 import { ActionRouter } from 'app/common/ActionRouter';
 import { BaseComponent, BaseLogger, createRpcLogger, PluginInstance, warnIfNotReady } from 'app/common/PluginInstance';
 import { tbind } from 'app/common/tbind';
@@ -41,7 +42,7 @@ import { getOriginUrl } from 'app/common/urlUtils';
 import { GristAPI, RPC_GRISTAPI_INTERFACE } from 'app/plugin/GristAPI';
 import { RenderOptions, RenderTarget } from 'app/plugin/RenderOptions';
 import { checkers } from 'app/plugin/TypeCheckers';
-import { Computed, dom as grainjsDom, Observable } from 'grainjs';
+import { dom as grainjsDom, Observable } from 'grainjs';
 import { IMsgCustom, IMsgRpcCall, IRpcLogger, MsgType, Rpc } from 'grain-rpc';
 import { Disposable } from './dispose';
 import isEqual from 'lodash/isEqual';
@@ -73,8 +74,6 @@ export class SafeBrowser extends BaseComponent {
       new IframeProcess(safeBrowser, rpc, src);
   }
 
-  public theme = this._options.theme;
-
   // All view processes. This is not used anymore to dispose all processes on deactivation (this is
   // now achieved using `this._mainProcess.autoDispose(...)`) but rather to be able to dispatch
   // events to all processes (such as doc actions which will need soon).
@@ -94,7 +93,6 @@ export class SafeBrowser extends BaseComponent {
     pluginInstance: PluginInstance,
     clientScope: ClientScope,
     untrustedContentOrigin: string,
-    theme: Computed<Theme>,
     mainPath?: string,
     baseLogger?: BaseLogger,
     rpcLogger?: IRpcLogger,
@@ -312,7 +310,7 @@ class IframeProcess extends ViewProcess {
     const listener = async (event: MessageEvent) => {
       if (event.source === iframe.contentWindow) {
         if (event.data.mtype === MsgType.Ready) {
-          await this._sendTheme({theme: safeBrowser.theme.get(), fromReady: true});
+          await this._sendTheme({theme: gristThemeObs().get(), fromReady: true});
         }
 
         if (event.data.data?.message === 'themeInitialized') {
@@ -328,15 +326,11 @@ class IframeProcess extends ViewProcess {
     });
     this.rpc.setSendMessage(msg => iframe.contentWindow!.postMessage(msg, '*'));
 
-    if (safeBrowser.theme) {
-      this.autoDispose(
-        safeBrowser.theme.addListener(async (newTheme, oldTheme) => {
-          if (isEqual(newTheme, oldTheme)) { return; }
+    this.autoDispose(gristThemeObs().addListener(async (newTheme, oldTheme) => {
+      if (isEqual(newTheme, oldTheme)) { return; }
 
-          await this._sendTheme({theme: newTheme});
-        })
-      );
-    }
+      await this._sendTheme({theme: newTheme});
+    }));
   }
 
   private async _sendTheme({theme, fromReady = false}: {theme: Theme, fromReady?: boolean}) {

app/client/lib/autocomplete.ts

@@ -4,7 +4,8 @@
 import {createPopper, Modifier, Instance as Popper, Options as PopperOptions} from '@popperjs/core';
 import {ACItem, ACResults, HighlightFunc} from 'app/client/lib/ACIndex';
 import {reportError} from 'app/client/models/errors';
-import {Disposable, dom, EventCB, IDisposable} from 'grainjs';
+import {testId, theme} from 'app/client/ui2018/cssVars';
+import {Disposable, dom, DomContents, EventCB, IDisposable} from 'grainjs';
 import {obsArray, onKeyElem, styled} from 'grainjs';
 import merge = require('lodash/merge');
 import maxSize from 'popper-max-size-modifier';
@@ -26,6 +27,9 @@ export interface IAutocompleteOptions<Item extends ACItem> {
   // Defaults to the document body.
   attach?: Element|string|null;
 
+  // If provided, builds and shows the message when there are no items (excluding any extra items).
+  buildNoItemsMessage?: () => DomContents;
+
   // Given a search term, return the list of Items to render.
   search(searchText: string): Promise<ACResults<Item>>;
 
@@ -46,7 +50,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
   // The UL element containing the actual menu items.
   protected _menuContent: HTMLElement;
 
-  // Index into _items as well as into _menuContent, -1 if nothing selected.
+  // Index into _menuContent, -1 if nothing selected.
   protected _selectedIndex: number = -1;
 
   // Currently selected element.
@@ -56,6 +60,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
   private _mouseOver: {reset(): void};
   private _lastAsTyped: string;
   private _items = this.autoDispose(obsArray<Item>([]));
+  private _extraItems = this.autoDispose(obsArray<Item>([]));
   private _highlightFunc: HighlightFunc;
 
   constructor(
@@ -65,14 +70,19 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
     super();
 
     const content = cssMenuWrap(
-      this._menuContent = cssMenu({class: _options.menuCssClass || ''},
-        dom.forEach(this._items, (item) => _options.renderItem(item, this._highlightFunc)),
+      cssMenu(
+        {class: _options.menuCssClass || ''},
         dom.style('min-width', _triggerElem.getBoundingClientRect().width + 'px'),
-        dom.on('mouseleave', (ev) => this._setSelected(-1, true)),
-        dom.on('click', (ev) => {
-          this._setSelected(this._findTargetItem(ev.target), true);
-          if (_options.onClick) { _options.onClick(); }
-        })
+        this._maybeShowNoItemsMessage(),
+        this._menuContent = dom('div',
+          dom.forEach(this._items, (item) => _options.renderItem(item, this._highlightFunc)),
+          dom.forEach(this._extraItems, (item) => _options.renderItem(item, this._highlightFunc)),
+          dom.on('mouseleave', (ev) => this._setSelected(-1, true)),
+          dom.on('click', (ev) => {
+            this._setSelected(this._findTargetItem(ev.target), true);
+            if (_options.onClick) { _options.onClick(); }
+          }),
+        ),
       ),
       // Prevent trigger element from being blurred on click.
       dom.on('mousedown', (ev) => ev.preventDefault()),
@@ -104,7 +114,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
   }
 
   public getSelectedItem(): Item|undefined {
-    return this._items.get()[this._selectedIndex];
+    return this._allItems[this._selectedIndex];
   }
 
   public search(findMatch?: (items: Item[]) => number) {
@@ -145,7 +155,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
 
   private _getNext(step: 1 | -1): number {
     // Pretend there is an extra element at the end to mean "nothing selected".
-    const xsize = this._items.get().length + 1;
+    const xsize = this._allItems.length + 1;
     const next = (this._selectedIndex + step + xsize) % xsize;
     return (next === xsize - 1) ? -1 : next;
   }
@@ -157,6 +167,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
     const acResults = await this._options.search(inputVal);
     this._highlightFunc = acResults.highlightFunc;
     this._items.set(acResults.items);
+    this._extraItems.set(acResults.extraItems);
 
     // Plain update() (which is deferred) may be better, but if _setSelected() causes scrolling
     // before the positions are updated, it causes the entire page to scroll horizontally.
@@ -166,12 +177,24 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
 
     let index: number;
     if (findMatch) {
-      index = findMatch(this._items.get());
+      index = findMatch(this._allItems);
     } else {
       index = inputVal ? acResults.selectIndex : -1;
     }
     this._setSelected(index, false);
   }
+
+  private get _allItems() {
+    return [...this._items.get(), ...this._extraItems.get()];
+  }
+
+  private _maybeShowNoItemsMessage() {
+    const {buildNoItemsMessage} = this._options;
+    if (!buildNoItemsMessage) { return null; }
+
+    return dom.maybe(use => use(this._items).length === 0, () =>
+      cssNoItemsMessage(buildNoItemsMessage(), testId('autocomplete-no-items-message')));
+  }
 }
 
 
@@ -253,3 +276,10 @@ const cssMenuWrap = styled('div', `
   flex-direction: column;
   outline: none;
 `);
+
+const cssNoItemsMessage = styled('div', `
+  color: ${theme.lightText};
+  padding: var(--weaseljs-menu-item-padding, 8px 24px);
+  text-align: center;
+  user-select: none;
+`);

app/client/lib/imports.d.ts

@@ -6,17 +6,20 @@ import * as GristDocModule from 'app/client/components/GristDoc';
 import * as ViewPane from 'app/client/components/ViewPane';
 import * as UserManagerModule from 'app/client/ui/UserManager';
 import * as searchModule from 'app/client/ui2018/search';
+import * as ace from 'ace-builds';
 import * as momentTimezone from 'moment-timezone';
 import * as plotly from 'plotly.js';
 
-export type PlotlyType = typeof plotly;
+export type Ace = typeof ace;
 export type MomentTimezone = typeof momentTimezone;
+export type PlotlyType = typeof plotly;
 
 export function loadAccountPage(): Promise<typeof AccountPageModule>;
 export function loadActivationPage(): Promise<typeof ActivationPageModule>;
 export function loadBillingPage(): Promise<typeof BillingPageModule>;
 export function loadAdminPanel(): Promise<typeof AdminPanelModule>;
 export function loadGristDoc(): Promise<typeof GristDocModule>;
+export function loadAce(): Promise<Ace>;
 export function loadMomentTimezone(): Promise<MomentTimezone>;
 export function loadPlotly(): Promise<PlotlyType>;
 export function loadSearch(): Promise<typeof searchModule>;

app/client/lib/imports.js

@@ -13,6 +13,17 @@ exports.loadAdminPanel = () => import('app/client/ui/AdminPanel' /* webpackChunk
 exports.loadGristDoc = () => import('app/client/components/GristDoc' /* webpackChunkName: "GristDoc" */);
 // When importing this way, the module is under the "default" member, not sure why (maybe
 // esbuild-loader's doing).
+exports.loadAce = () => import('ace-builds')
+  .then(async (m) => {
+    await Promise.all([
+      import('ace-builds/src-noconflict/ext-static_highlight'),
+      import('ace-builds/src-noconflict/mode-python'),
+      import('ace-builds/src-noconflict/theme-chrome'),
+      import('ace-builds/src-noconflict/theme-dracula'),
+    ]);
+
+    return m.default;
+  });
 exports.loadMomentTimezone = () => import('moment-timezone').then(m => m.default);
 exports.loadPlotly = () => import('plotly.js-basic-dist' /* webpackChunkName: "plotly" */);
 exports.loadSearch = () => import('app/client/ui2018/search' /* webpackChunkName: "search" */);

app/client/lib/nameUtils.ts

@@ -0,0 +1,14 @@
+/**
+ * We allow alphanumeric characters and certain common whitelisted characters (except at the start),
+ * plus everything non-ASCII (for non-English alphabets, which we want to allow but it's hard to be
+ * more precise about what exactly to allow).
+ */
+// eslint-disable-next-line no-control-regex
+const VALID_NAME_REGEXP = /^(\w|[^\u0000-\u007F])(\w|[- ./'"()]|[^\u0000-\u007F])*$/;
+
+/**
+ * Test name against various rules to check if it is a valid username.
+ */
+export function checkName(name: string): boolean {
+  return VALID_NAME_REGEXP.test(name);
+}

app/client/lib/timeUtils.ts

@@ -0,0 +1,21 @@
+import moment from 'moment';
+
+/**
+ * Given a UTC Date ISO 8601 string (the doc updatedAt string), gives a reader-friendly
+ * relative time to now - e.g. 'yesterday', '2 days ago'.
+ */
+export function getTimeFromNow(utcDateISO: string): string {
+  const time = moment.utc(utcDateISO);
+  const now = moment();
+  const diff = now.diff(time, 's');
+  if (diff < 0 && diff > -60) {
+    // If the time appears to be in the future, but less than a minute
+    // in the future, chalk it up to a difference in time
+    // synchronization and don't claim the resource will be changed in
+    // the future.  For larger differences, just report them
+    // literally, there's a more serious problem or lack of
+    // synchronization.
+    return now.fromNow();
+  }
+  return time.fromNow();
+}