(core) Add dropdown conditions

Summary:
Dropdown conditions let you specify a predicate formula that's used to filter
choices and references in their respective autocomplete dropdown menus.

Test Plan: Python and browser tests (WIP).

Reviewers: jarek, paulfitz

Reviewed By: jarek

Subscribers: dsagal, paulfitz

Differential Revision: https://phab.getgrist.com/D4235

app/client/aclui/ACLFormulaEditor.ts

@@ -1,13 +1,12 @@
 import ace, {Ace} from 'ace-builds';
 import {setupAceEditorCompletions} from 'app/client/components/AceEditorCompletions';
 import {theme} from 'app/client/ui2018/cssVars';
+import {gristThemeObs} from 'app/client/ui2018/theme';
 import {Theme} from 'app/common/ThemePrefs';
-import {getGristConfig} from 'app/common/urlUtils';
+import {dom, DomArg, Observable, styled} from 'grainjs';
-import {Computed, dom, DomArg, Listener, Observable, styled} from 'grainjs';
 import debounce from 'lodash/debounce';
 
 export interface ACLFormulaOptions {
-  gristTheme: Computed<Theme>;
   initialValue: string;
   readOnly: boolean;
   placeholder: DomArg;
@@ -22,19 +21,15 @@ export function aclFormulaEditor(options: ACLFormulaOptions) {
   const editor: Ace.Editor = ace.edit(editorElem);
 
   // Set various editor options.
-  function setAceTheme(gristTheme: Theme) {
+  function setAceTheme(newTheme: Theme) {
-    const {enableCustomCss} = getGristConfig();
+    const {appearance} = newTheme;
-    const gristAppearance = gristTheme.appearance;
+    const aceTheme = appearance === 'dark' ? 'dracula' : 'chrome';
-    const aceTheme = gristAppearance === 'dark' && !enableCustomCss ? 'dracula' : 'chrome';
     editor.setTheme(`ace/theme/${aceTheme}`);
   }
-  setAceTheme(options.gristTheme.get());
+  setAceTheme(gristThemeObs().get());
-  let themeListener: Listener | undefined;
+  const themeListener = gristThemeObs().addListener((newTheme) => {
-  if (!getGristConfig().enableCustomCss) {
+    setAceTheme(newTheme);
-    themeListener = options.gristTheme.addListener((gristTheme) => {
+  });
-      setAceTheme(gristTheme);
-    });
-  }
   // ACE editor resizes automatically when maxLines is set.
   editor.setOptions({enableLiveAutocompletion: true, maxLines: 10});
   editor.renderer.setShowGutter(false);       // Default line numbers to hidden

app/client/aclui/AccessRules.ts

@@ -36,14 +36,13 @@ import {ACLRuleCollection, isSchemaEditResource, SPECIAL_RULES_TABLE_ID} from 'a
 import {AclRuleProblem, AclTableDescription, getTableTitle} from 'app/common/ActiveDocAPI';
 import {BulkColValues, getColValues, RowRecord, UserAction} from 'app/common/DocActions';
 import {
-  FormulaProperties,
-  getFormulaProperties,
   RulePart,
   RuleSet,
   UserAttributeRule
 } from 'app/common/GranularAccessClause';
 import {isHiddenCol} from 'app/common/gristTypes';
 import {isNonNullish, unwrap} from 'app/common/gutil';
+import {getPredicateFormulaProperties, PredicateFormulaProperties} from 'app/common/PredicateFormula';
 import {SchemaTypes} from 'app/common/schema';
 import {MetaRowRecord} from 'app/common/TableData';
 import {
@@ -496,7 +495,7 @@ export class AccessRules extends Disposable {
     removeItem(this._userAttrRules, userAttr);
   }
 
-  public async checkAclFormula(text: string): Promise<FormulaProperties> {
+  public async checkAclFormula(text: string): Promise<PredicateFormulaProperties> {
     if (text) {
       return this.gristDoc.docComm.checkAclFormula(text);
     }
@@ -1465,7 +1464,6 @@ class ObsUserAttributeRule extends Disposable {
         cssColumnGroup(
           cssCell1(
             aclFormulaEditor({
-              gristTheme: this._accessRules.gristDoc.currentTheme,
               initialValue: this._charId.get(),
               readOnly: false,
               setValue: (text) => this._setUserAttr(text),
@@ -1598,7 +1596,8 @@ class ObsRulePart extends Disposable {
   // If the formula failed validation, the error message to show. Blank if valid.
   private _formulaError = Observable.create(this, '');
 
-  private _formulaProperties = Observable.create<FormulaProperties>(this, getAclFormulaProperties(this._rulePart));
+  private _formulaProperties = Observable.create<PredicateFormulaProperties>(this,
+    getAclFormulaProperties(this._rulePart));
 
   // Error message if any validation failed.
   private _error: Computed<string>;
@@ -1618,7 +1617,7 @@ class ObsRulePart extends Disposable {
 
     this._error = Computed.create(this, (use) => {
       return use(this._formulaError) ||
-        this._warnInvalidColIds(use(this._formulaProperties).usedColIds) ||
+        this._warnInvalidColIds(use(this._formulaProperties).recColIds) ||
         ( !this._ruleSet.isLastCondition(use, this) &&
           use(this._aclFormula) === '' &&
           permissionSetToText(use(this._permissions)) !== '' ?
@@ -1690,7 +1689,6 @@ class ObsRulePart extends Disposable {
         cssCell2(
           wide ? cssCell4.cls('') : null,
           aclFormulaEditor({
-            gristTheme: this._ruleSet.accessRules.gristDoc.currentTheme,
             initialValue: this._aclFormula.get(),
             readOnly: this.isBuiltIn(),
             setValue: (value) => this._setAclFormula(value),
@@ -1913,9 +1911,9 @@ function getChangedStatus(value: boolean): RuleStatus {
   return value ? RuleStatus.ChangedValid : RuleStatus.Unchanged;
 }
 
-function getAclFormulaProperties(part?: RulePart): FormulaProperties {
+function getAclFormulaProperties(part?: RulePart): PredicateFormulaProperties {
   const aclFormulaParsed = part?.origRecord?.aclFormulaParsed;
-  return aclFormulaParsed ? getFormulaProperties(JSON.parse(String(aclFormulaParsed))) : {};
+  return aclFormulaParsed ? getPredicateFormulaProperties(JSON.parse(String(aclFormulaParsed))) : {};
 }
 
 // Return a rule set if it applies to one of the specified columns.

app/client/components/AceEditor.js

@@ -7,10 +7,10 @@ require('ace-builds/src-noconflict/theme-chrome');
 require('ace-builds/src-noconflict/theme-dracula');
 require('ace-builds/src-noconflict/ext-language_tools');
 var {setupAceEditorCompletions} = require('./AceEditorCompletions');
-var {getGristConfig} = require('../../common/urlUtils');
 var dom = require('../lib/dom');
 var dispose = require('../lib/dispose');
 var modelUtil = require('../models/modelUtil');
+var {gristThemeObs} = require('../ui2018/theme');
 
 /**
  * A class to help set up the ace editor with standard formatting and convenience functions
@@ -28,10 +28,9 @@ function AceEditor(options) {
   this.observable = options.observable || null;
   this.saveValueOnBlurEvent = !(options.saveValueOnBlurEvent === false);
   this.calcSize = options.calcSize || ((_elem, size) => size);
-  this.gristDoc = options.gristDoc || null;
-  this.column = options.column || null;
   this.editorState = options.editorState || null;
   this._readonly = options.readonly || false;
+  this._getSuggestions = options.getSuggestions || null;
 
   this.editor = null;
   this.editorDom = null;
@@ -185,19 +184,8 @@ AceEditor.prototype.setFontSize = function(pxVal) {
 AceEditor.prototype._setup = function() {
   // Standard editor setup
   this.editor = this.autoDisposeWith('destroy', ace.edit(this.editorDom));
-  if (this.gristDoc && this.column) {
+  if (this._getSuggestions) {
-    const getSuggestions = (prefix) => {
+    setupAceEditorCompletions(this.editor, {getSuggestions: this._getSuggestions});
-      const section = this.gristDoc.viewModel.activeSection();
-      // If section is disposed or is pointing to an empty row, don't try to autocomplete.
-      if (!section?.getRowId()) {
-        return [];
-      }
-      const tableId = section.table().tableId();
-      const columnId = this.column.colId();
-      const rowId = section.activeRowId();
-      return this.gristDoc.docComm.autocomplete(prefix, tableId, columnId, rowId);
-    };
-    setupAceEditorCompletions(this.editor, {getSuggestions});
   }
   this.editor.setOptions({
     enableLiveAutocompletion: true,   // use autocompletion without needing special activation.
@@ -205,13 +193,10 @@ AceEditor.prototype._setup = function() {
   this.session = this.editor.getSession();
   this.session.setMode('ace/mode/python');
 
-  const gristTheme = this.gristDoc?.currentTheme;
+  this._setAceTheme(gristThemeObs().get());
-  this._setAceTheme(gristTheme?.get());
+  this.autoDispose(gristThemeObs().addListener((newTheme) => {
-  if (!getGristConfig().enableCustomCss && gristTheme) {
+    this._setAceTheme(newTheme);
-    this.autoDispose(gristTheme.addListener((theme) => {
+  }));
-      this._setAceTheme(theme);
-    }));
-  }
 
   // Default line numbers to hidden
   this.editor.renderer.setShowGutter(false);
@@ -283,10 +268,9 @@ AceEditor.prototype._getContentHeight = function() {
   return Math.max(1, this.session.getScreenLength()) * this.editor.renderer.lineHeight;
 };
 
-AceEditor.prototype._setAceTheme = function(gristTheme) {
+AceEditor.prototype._setAceTheme = function(newTheme) {
-  const {enableCustomCss} = getGristConfig();
+  const {appearance} = newTheme;
-  const gristAppearance = gristTheme?.appearance;
+  const aceTheme = appearance === 'dark' ? 'dracula' : 'chrome';
-  const aceTheme = gristAppearance === 'dark' && !enableCustomCss ? 'dracula' : 'chrome';
   this.editor.setTheme(`ace/theme/${aceTheme}`);
 };
 

app/client/components/ChartView.ts

@@ -17,6 +17,7 @@ import {cssFieldEntry, cssFieldLabel, IField, VisibleFieldsConfig } from 'app/cl
 import {IconName} from 'app/client/ui2018/IconList';
 import {squareCheckbox} from 'app/client/ui2018/checkbox';
 import {theme, vars} from 'app/client/ui2018/cssVars';
+import {gristThemeObs} from 'app/client/ui2018/theme';
 import {cssDragger} from 'app/client/ui2018/draggableList';
 import {icon} from 'app/client/ui2018/icons';
 import {IOptionFull, linkSelect, menu, menuItem, menuText, select} from 'app/client/ui2018/menus';
@@ -229,7 +230,7 @@ export class ChartView extends Disposable {
     this.listenTo(this.sortedRows, 'rowNotify', this._update);
     this.autoDispose(this.sortedRows.getKoArray().subscribe(this._update));
     this.autoDispose(this._formatterComp.subscribe(this._update));
-    this.autoDispose(this.gristDoc.currentTheme.addListener(() => this._update()));
+    this.autoDispose(gristThemeObs().addListener(() => this._update()));
   }
 
   public prepareToPrint(onOff: boolean) {
@@ -387,8 +388,7 @@ export class ChartView extends Disposable {
   }
 
   private _getPlotlyTheme(): Partial<Layout> {
-    const appModel = this.gristDoc.docPageModel.appModel;
+    const {colors} = gristThemeObs().get();
-    const {colors} = appModel.currentTheme.get();
     return {
       paper_bgcolor: colors['chart-bg'],
       plot_bgcolor: colors['chart-bg'],

app/client/components/ColumnTransform.ts

@@ -91,9 +91,9 @@ export class ColumnTransform extends Disposable {
   protected buildEditorDom(optInit?: string) {
     if (!this.editor) {
       this.editor = this.autoDispose(AceEditor.create({
-        gristDoc: this.gristDoc,
         observable: this.transformColumn.formula,
         saveValueOnBlurEvent: false,
+        // TODO: set `getSuggestions` (see `FormulaEditor.ts` for an example).
       }));
     }
     return this.editor.buildDom((aceObj: any) => {

app/client/components/CustomView.ts

@@ -321,7 +321,7 @@ export class CustomView extends Disposable {
           }),
           new MinimumLevel(AccessLevel.none)); // none access is enough
         frame.useEvents(
-          ThemeNotifier.create(frame, this.gristDoc.currentTheme),
+          ThemeNotifier.create(frame),
           new MinimumLevel(AccessLevel.none));
       },
       onElem: (iframe) => onFrameFocus(iframe, () => {

app/client/components/DropdownConditionConfig.ts

@@ -0,0 +1,197 @@
+import {buildDropdownConditionEditor} from 'app/client/components/DropdownConditionEditor';
+import {makeT} from 'app/client/lib/localization';
+import {ViewFieldRec} from 'app/client/models/DocModel';
+import {cssLabel, cssRow} from 'app/client/ui/RightPanelStyles';
+import {withInfoTooltip} from 'app/client/ui/tooltips';
+import {textButton } from 'app/client/ui2018/buttons';
+import {testId, theme} from 'app/client/ui2018/cssVars';
+import {ISuggestionWithValue} from 'app/common/ActiveDocAPI';
+import {getPredicateFormulaProperties} from 'app/common/PredicateFormula';
+import {Computed, Disposable, dom, Observable, styled} from 'grainjs';
+
+const t = makeT('DropdownConditionConfig');
+
+/**
+ * Right panel configuration for dropdown conditions.
+ *
+ * Contains an instance of `DropdownConditionEditor`, the class responsible
+ * for setting dropdown conditions.
+ */
+export class DropdownConditionConfig extends Disposable {
+  private _text = Computed.create(this, use => {
+    const dropdownCondition = use(this._field.dropdownCondition);
+    if (!dropdownCondition) { return ''; }
+
+    return dropdownCondition.text;
+  });
+
+  private _saveError = Observable.create<string | null>(this, null);
+
+  private _properties = Computed.create(this, use => {
+    const dropdownCondition = use(this._field.dropdownCondition);
+    if (!dropdownCondition?.parsed) { return null; }
+
+    return getPredicateFormulaProperties(JSON.parse(dropdownCondition.parsed));
+  });
+
+  private _column = Computed.create(this, use => use(this._field.column));
+
+  private _columns = Computed.create(this, use => use(use(use(this._column).table).visibleColumns));
+
+  private _refColumns = Computed.create(this, use => {
+    const refTable = use(use(this._column).refTable);
+    if (!refTable) { return null; }
+
+    return use(refTable.visibleColumns);
+  });
+
+  private _propertiesError = Computed.create<string | null>(this, use => {
+    const properties = use(this._properties);
+    if (!properties) { return null; }
+
+    const {recColIds = [], choiceColIds = []} = properties;
+    const columns = use(this._columns);
+    const validRecColIds = new Set(columns.map((({colId}) => use(colId))));
+    const invalidRecColIds = recColIds.filter(colId => !validRecColIds.has(colId));
+    if (invalidRecColIds.length > 0) {
+      return t('Invalid columns: {{colIds}}', {colIds: invalidRecColIds.join(', ')});
+    }
+
+    const refColumns = use(this._refColumns);
+    if (refColumns) {
+      const validChoiceColIds = new Set(['id', ...refColumns.map((({colId}) => use(colId)))]);
+      const invalidChoiceColIds = choiceColIds.filter(colId => !validChoiceColIds.has(colId));
+      if (invalidChoiceColIds.length > 0) {
+        return t('Invalid columns: {{colIds}}', {colIds: invalidChoiceColIds.join(', ')});
+      }
+    }
+
+    return null;
+  });
+
+  private _error = Computed.create<string | null>(this, (use) => {
+    const maybeSaveError = use(this._saveError);
+    if (maybeSaveError) { return maybeSaveError; }
+
+    const maybeCompiled = use(this._field.dropdownConditionCompiled);
+    if (maybeCompiled?.kind === 'failure') { return maybeCompiled.error; }
+
+    const maybePropertiesError = use(this._propertiesError);
+    if (maybePropertiesError) { return maybePropertiesError; }
+
+    return null;
+  });
+
+  private _disabled = Computed.create(this, use =>
+    use(this._field.disableModify) ||
+    use(use(this._column).disableEditData) ||
+    use(this._field.config.multiselect)
+  );
+
+  private _isEditingCondition = Observable.create(this, false);
+
+  private _isRefField = Computed.create(this, (use) =>
+    ['Ref', 'RefList'].includes(use(use(this._column).pureType)));
+
+  private _tooltip = Computed.create(this, use => use(this._isRefField)
+    ? 'setRefDropdownCondition'
+    : 'setChoiceDropdownCondition');
+
+  private _editorElement: HTMLElement;
+
+  constructor(private _field: ViewFieldRec) {
+    super();
+
+    this.autoDispose(this._text.addListener(() => {
+      this._saveError.set('');
+    }));
+  }
+
+  public buildDom() {
+    return [
+      dom.maybe((use) => !(use(this._isEditingCondition) || Boolean(use(this._text))), () => [
+        cssSetDropdownConditionRow(
+          dom.domComputed(use => withInfoTooltip(
+            textButton(
+              t('Set dropdown condition'),
+              dom.on('click', () => {
+                this._isEditingCondition.set(true);
+                setTimeout(() => this._editorElement.focus(), 0);
+              }),
+              dom.prop('disabled', this._disabled),
+              testId('field-set-dropdown-condition'),
+            ),
+            use(this._tooltip),
+          )),
+        ),
+      ]),
+      dom.maybe((use) => use(this._isEditingCondition) || Boolean(use(this._text)), () => [
+        cssLabel(t('Dropdown Condition')),
+        cssRow(
+          dom.create(buildDropdownConditionEditor,
+            {
+              value: this._text,
+              disabled: this._disabled,
+              getAutocompleteSuggestions: () => this._getAutocompleteSuggestions(),
+              onSave: async (value) => {
+                try {
+                  const widgetOptions = this._field.widgetOptionsJson.peek();
+                  if (value.trim() === '') {
+                    delete widgetOptions.dropdownCondition;
+                  } else {
+                    widgetOptions.dropdownCondition = {text: value};
+                  }
+                  await this._field.widgetOptionsJson.setAndSave(widgetOptions);
+                } catch (e) {
+                  if (e?.code === 'ACL_DENY') {
+                    reportError(e);
+                  } else {
+                    this._saveError.set(e.message.replace(/^\[Sandbox\]/, '').trim());
+                  }
+                }
+              },
+              onDispose: () => {
+                this._isEditingCondition.set(false);
+              },
+            },
+            (el) => { this._editorElement = el; },
+            testId('field-dropdown-condition'),
+          ),
+        ),
+        dom.maybe(this._error, (error) => cssRow(
+          cssDropdownConditionError(error), testId('field-dropdown-condition-error')),
+        ),
+      ]),
+    ];
+  }
+
+  private _getAutocompleteSuggestions(): ISuggestionWithValue[] {
+    const variables = ['choice'];
+    const refColumns = this._refColumns.get();
+    if (refColumns) {
+      variables.push('choice.id', ...refColumns.map(({colId}) => `choice.${colId.peek()}`));
+    }
+    const columns = this._columns.get();
+    variables.push(
+      ...columns.map(({colId}) => `$${colId.peek()}`),
+      ...columns.map(({colId}) => `rec.${colId.peek()}`),
+    );
+
+    const suggestions = [
+      'and', 'or', 'not', 'in', 'is', 'True', 'False', 'None',
+      'OWNER', 'EDITOR', 'VIEWER',
+      ...variables,
+    ];
+    return suggestions.map(suggestion => [suggestion, null]);
+  }
+}
+
+const cssSetDropdownConditionRow = styled(cssRow, `
+  margin-top: 16px;
+`);
+
+const cssDropdownConditionError = styled('div', `
+  color: ${theme.errorText};
+  margin-top: 4px;
+  width: 100%;
+`);

app/client/components/DropdownConditionEditor.ts

@@ -0,0 +1,234 @@
+import * as AceEditor from 'app/client/components/AceEditor';
+import {createGroup} from 'app/client/components/commands';
+import {makeT} from 'app/client/lib/localization';
+import {buildHighlightedCode} from 'app/client/ui/CodeHighlight';
+import {theme} from 'app/client/ui2018/cssVars';
+import {createMobileButtons, getButtonMargins} from 'app/client/widgets/EditorButtons';
+import {EditorPlacement, ISize} from 'app/client/widgets/EditorPlacement';
+import {initializeAceOptions} from 'app/client/widgets/FormulaEditor';
+import {IEditorCommandGroup} from 'app/client/widgets/NewBaseEditor';
+import {ISuggestionWithValue} from 'app/common/ActiveDocAPI';
+import {
+  Computed,
+  Disposable,
+  dom,
+  DomElementArg,
+  Holder,
+  IDisposableOwner,
+  Observable,
+  styled,
+} from 'grainjs';
+
+const t = makeT('DropdownConditionEditor');
+
+interface BuildDropdownConditionEditorOptions {
+  value: Computed<string>;
+  disabled: Computed<boolean>;
+  onSave(value: string): Promise<void>;
+  onDispose(): void;
+  getAutocompleteSuggestions(prefix: string): ISuggestionWithValue[];
+}
+
+/**
+ * Builds an editor for dropdown conditions.
+ *
+ * Dropdown conditions are client-evaluated predicate formulas used to filter
+ * items shown in autocomplete dropdowns for Choice and Reference type columns.
+ *
+ * Unlike Python formulas, dropdown conditions only support a very limited set of
+ * features. They are a close relative of ACL formulas, sharing the same underlying
+ * parser and compiler.
+ *
+ * See `sandbox/grist/predicate_formula.py` and `app/common/PredicateFormula.ts` for
+ * more details on parsing and compiling, respectively.
+ */
+export function buildDropdownConditionEditor(
+  owner: IDisposableOwner,
+  options: BuildDropdownConditionEditorOptions,
+  ...args: DomElementArg[]
+) {
+  const {value, disabled, onSave, onDispose, getAutocompleteSuggestions} = options;
+  return dom.create(buildHighlightedCode,
+    value,
+    {maxLines: 1},
+    dom.cls(cssDropdownConditionField.className),
+    dom.cls('disabled'),
+    cssDropdownConditionField.cls('-disabled', disabled),
+    {tabIndex: '-1'},
+    dom.on('focus', (_, refElem) => openDropdownConditionEditor(owner, {
+      refElem,
+      value,
+      onSave,
+      onDispose,
+      getAutocompleteSuggestions,
+    })),
+    ...args,
+  );
+}
+
+function openDropdownConditionEditor(owner: IDisposableOwner, options: {
+  refElem: Element;
+  value: Computed<string>;
+  onSave: (value: string) => Promise<void>;
+  onDispose: () => void;
+  getAutocompleteSuggestions(prefix: string): ISuggestionWithValue[];
+}) {
+  const {refElem, value, onSave, onDispose, getAutocompleteSuggestions} = options;
+
+  const saveAndDispose = async () => {
+    const editorValue = editor.getValue();
+    if (editorValue !== value.get()) {
+      await onSave(editorValue);
+    }
+    if (editor.isDisposed()) { return; }
+
+    editor.dispose();
+  };
+
+  const commands: IEditorCommandGroup = {
+    fieldEditCancel: () => editor.dispose(),
+    fieldEditSaveHere: () => editor.blur(),
+    fieldEditSave: () => editor.blur(),
+  };
+
+  const editor = DropdownConditionEditor.create(owner, {
+    editValue: value.get(),
+    commands,
+    onBlur: saveAndDispose,
+    getAutocompleteSuggestions,
+  });
+  editor.attach(refElem);
+  editor.onDispose(() => onDispose());
+}
+
+interface DropdownConditionEditorOptions {
+  editValue: string;
+  commands: IEditorCommandGroup;
+  onBlur(): Promise<void>;
+  getAutocompleteSuggestions(prefix: string): ISuggestionWithValue[];
+}
+
+class DropdownConditionEditor extends Disposable {
+  private _aceEditor: any;
+  private _dom: HTMLElement;
+  private _editorPlacement!: EditorPlacement;
+  private _placementHolder = Holder.create(this);
+  private _isEmpty: Computed<boolean>;
+
+  constructor(private _options: DropdownConditionEditorOptions) {
+    super();
+
+    const initialValue = _options.editValue;
+    const editorState = Observable.create(this, initialValue);
+
+    this._aceEditor = this.autoDispose(AceEditor.create({
+      calcSize: this._calcSize.bind(this),
+      editorState,
+      getSuggestions: _options.getAutocompleteSuggestions,
+    }));
+
+    this._isEmpty = Computed.create(this, editorState, (_use, state) => state === '');
+    this.autoDispose(this._isEmpty.addListener(() => this._updateEditorPlaceholder()));
+
+    const commandGroup = this.autoDispose(createGroup({
+      ..._options.commands,
+    }, this, true));
+
+    this._dom = cssDropdownConditionEditorWrapper(
+      cssDropdownConditionEditor(
+        createMobileButtons(_options.commands),
+        this._aceEditor.buildDom((aceObj: any) => {
+          initializeAceOptions(aceObj);
+          const val = initialValue;
+          const pos = val.length;
+          this._aceEditor.setValue(val, pos);
+          this._aceEditor.attachCommandGroup(commandGroup);
+          if (val === '') {
+            this._updateEditorPlaceholder();
+          }
+        })
+      ),
+    );
+  }
+
+  public attach(cellElem: Element): void {
+    this._editorPlacement = EditorPlacement.create(this._placementHolder, this._dom, cellElem, {
+      margins: getButtonMargins(),
+    });
+    this.autoDispose(this._editorPlacement.onReposition.addListener(this._aceEditor.resize, this._aceEditor));
+    this._aceEditor.onAttach();
+    this._updateEditorPlaceholder();
+    this._aceEditor.resize();
+    this._aceEditor.getEditor().focus();
+    this._aceEditor.getEditor().on('blur', () => this._options.onBlur());
+  }
+
+  public getValue(): string {
+    return this._aceEditor.getValue();
+  }
+
+  public blur() {
+    this._aceEditor.getEditor().blur();
+  }
+
+  private _updateEditorPlaceholder() {
+    const editor = this._aceEditor.getEditor();
+    const shouldShowPlaceholder = editor.session.getValue().length === 0;
+    if (editor.renderer.emptyMessageNode) {
+      // Remove the current placeholder if one is present.
+      editor.renderer.scroller.removeChild(editor.renderer.emptyMessageNode);
+    }
+    if (!shouldShowPlaceholder) {
+      editor.renderer.emptyMessageNode = null;
+    } else {
+      editor.renderer.emptyMessageNode = cssDropdownConditionPlaceholder(t('Enter condition.'));
+      editor.renderer.scroller.appendChild(editor.renderer.emptyMessageNode);
+    }
+  }
+
+  private _calcSize(elem: HTMLElement, desiredElemSize: ISize) {
+    const placeholder: HTMLElement | undefined = this._aceEditor.getEditor().renderer.emptyMessageNode;
+    if (placeholder) {
+      return this._editorPlacement.calcSizeWithPadding(elem, {
+        width: placeholder.scrollWidth,
+        height: placeholder.scrollHeight,
+      });
+    } else {
+      return this._editorPlacement.calcSizeWithPadding(elem, {
+        width: desiredElemSize.width,
+        height: desiredElemSize.height,
+      });
+    }
+  }
+}
+
+const cssDropdownConditionField = styled('div', `
+  flex: auto;
+  cursor: pointer;
+  margin-top: 4px;
+
+  &-disabled {
+    opacity: 0.4;
+    pointer-events: none;
+  }
+`);
+
+const cssDropdownConditionEditorWrapper = styled('div.default_editor.formula_editor_wrapper', `
+  border-radius: 3px;
+`);
+
+const cssDropdownConditionEditor = styled('div', `
+  background-color: ${theme.aceEditorBg};
+  padding: 5px;
+  z-index: 10;
+  overflow: hidden;
+  flex: none;
+  min-height: 22px;
+  border-radius: 3px;
+`);
+
+const cssDropdownConditionPlaceholder = styled('div', `
+  color: ${theme.lightText};
+  font-style: italic;
+  white-space: nowrap;
+`);

app/client/components/GristDoc.ts

@@ -192,8 +192,6 @@ export class GristDoc extends DisposableWithEvents {
   // Holder for the popped up formula editor.
   public readonly formulaPopup = Holder.create(this);
 
-  public readonly currentTheme = this.docPageModel.appModel.currentTheme;
-
   public get docApi() {
     return this.docPageModel.appModel.api.getDocAPI(this.docPageModel.currentDocId.get()!);
   }
@@ -238,7 +236,6 @@ export class GristDoc extends DisposableWithEvents {
       untrustedContentOrigin: app.topAppModel.getUntrustedContentOrigin(),
       docComm: this.docComm,
       clientScope: app.clientScope,
-      theme: this.currentTheme,
     });
 
     // Maintain the MetaRowModel for the global document info, including docId and peers.

app/client/components/Importer.ts

@@ -1345,8 +1345,9 @@ export class Importer extends DisposableWithEvents {
       const column = use(field.column);
       return use(column.formula);
     });
-    const codeOptions = {gristTheme: this._gristDoc.currentTheme, placeholder: 'Skip', maxLines: 1};
+    const codeOptions = {placeholder: 'Skip', maxLines: 1};
-    return cssFieldFormula(formula, codeOptions,
+    return dom.create(buildHighlightedCode, formula, codeOptions,
+      dom.cls(cssFieldFormula.className),
       dom.cls('disabled'),
       dom.cls('formula_field_sidepane'),
       {tabIndex: '-1'},
@@ -1701,7 +1702,7 @@ const cssColumnMatchRow = styled('div', `
   }
 `);
 
-const cssFieldFormula = styled(buildHighlightedCode, `
+const cssFieldFormula = styled('div', `
   flex: auto;
   cursor: pointer;
   margin-top: 1px;

app/client/components/Printing.ts

@@ -2,7 +2,7 @@ import {CustomView} from 'app/client/components/CustomView';
 import {DataRowModel} from 'app/client/models/DataRowModel';
 import DataTableModel from 'app/client/models/DataTableModel';
 import {ViewSectionRec} from 'app/client/models/DocModel';
-import {prefersDarkMode, prefersDarkModeObs} from 'app/client/ui2018/cssVars';
+import {prefersColorSchemeDark, prefersColorSchemeDarkObs} from 'app/client/ui2018/theme';
 import {dom} from 'grainjs';
 
 type RowId = number|'new';
@@ -45,7 +45,7 @@ export async function printViewSection(layout: any, viewSection: ViewSectionRec)
     // is Grist temporarily reverting to the light theme until the print dialog is dismissed.
     // As a workaround, we'll temporarily pause our listener, and unpause after the print dialog
     // is dismissed.
-    prefersDarkModeObs().pause();
+    prefersColorSchemeDarkObs().pause();
 
     // Hide all layout boxes that do NOT contain the section to be printed.
     layout?.forEachBox((box: any) => {
@@ -87,10 +87,10 @@ export async function printViewSection(layout: any, viewSection: ViewSectionRec)
       prepareToPrint(false);
     }
     delete (window as any).afterPrintCallback;
-    prefersDarkModeObs().pause(false);
+    prefersColorSchemeDarkObs().pause(false);
 
     // This may have changed while window.print() was blocking.
-    prefersDarkModeObs().set(prefersDarkMode());
+    prefersColorSchemeDarkObs().set(prefersColorSchemeDark());
   });
 
   // Running print on a timeout makes it possible to test printing using selenium, and doesn't

app/client/components/WidgetFrame.ts

@@ -7,11 +7,11 @@ import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
 import {makeTestId} from 'app/client/lib/domUtils';
 import {ColumnRec, ViewSectionRec} from 'app/client/models/DocModel';
 import {reportError} from 'app/client/models/errors';
+import {gristThemeObs} from 'app/client/ui2018/theme';
 import {AccessLevel, ICustomWidget, isSatisfied, matchWidget} from 'app/common/CustomWidget';
 import {DisposableWithEvents} from 'app/common/DisposableWithEvents';
 import {BulkColValues, fromTableDataAction, RowRecord} from 'app/common/DocActions';
 import {extractInfoFromColType, reencodeAsAny} from 'app/common/gristTypes';
-import {Theme} from 'app/common/ThemePrefs';
 import {getGristConfig} from 'app/common/urlUtils';
 import {
   AccessTokenOptions, CursorPos, CustomSectionAPI, FetchSelectedOptions, GristDocAPI, GristView,
@@ -696,10 +696,10 @@ export class ConfigNotifier extends BaseEventSource {
  * Notifies about theme changes. Exposed in the API as `onThemeChange`.
  */
 export class ThemeNotifier extends BaseEventSource {
-  constructor(private _theme: Computed<Theme>) {
+  constructor() {
     super();
     this.autoDispose(
-      this._theme.addListener((newTheme, oldTheme) => {
+      gristThemeObs().addListener((newTheme, oldTheme) => {
         if (isEqual(newTheme, oldTheme)) { return; }
 
         this._update();
@@ -715,7 +715,7 @@ export class ThemeNotifier extends BaseEventSource {
     if (this.isDisposed()) { return; }
 
     this._notify({
-      theme: this._theme.get(),
+      theme: gristThemeObs().get(),
       fromReady,
     });
   }

app/client/lib/ACIndex.ts

@@ -51,6 +51,9 @@ export interface ACResults<Item extends ACItem> {
   // Matching items in order from best match to worst.
   items: Item[];
 
+  // Additional items to show (e.g. the "Add New" item, for Choice and Reference fields).
+  extraItems: Item[];
+
   // May be used to highlight matches using buildHighlightedDom().
   highlightFunc: HighlightFunc;
 
@@ -159,7 +162,7 @@ export class ACIndexImpl<Item extends ACItem> implements ACIndex<Item> {
 
     if (!cleanedSearchText) {
       // In this case we are just returning the first few items.
-      return {items, highlightFunc: highlightNone, selectIndex: -1};
+      return {items, extraItems: [], highlightFunc: highlightNone, selectIndex: -1};
     }
 
     const highlightFunc = highlightMatches.bind(null, searchWords);
@@ -170,7 +173,7 @@ export class ACIndexImpl<Item extends ACItem> implements ACIndex<Item> {
     if (selectIndex >= 0 && !startsWithText(items[selectIndex], cleanedSearchText, searchWords)) {
       selectIndex = -1;
     }
-    return {items, highlightFunc, selectIndex};
+    return {items, extraItems: [], highlightFunc, selectIndex};
   }
 
   /**

app/client/lib/ACUserManager.ts

@@ -98,7 +98,7 @@ export function buildACMemberEmail(
         label: text,
         id: 0,
       };
-      results.items.push(newObject);
+      results.extraItems.push(newObject);
     }
     return results;
   };

app/client/lib/DocPluginManager.ts

@@ -4,8 +4,6 @@ import {SafeBrowser} from 'app/client/lib/SafeBrowser';
 import {ActiveDocAPI} from 'app/common/ActiveDocAPI';
 import {LocalPlugin} from 'app/common/plugin';
 import {createRpcLogger, PluginInstance} from 'app/common/PluginInstance';
-import {Theme} from 'app/common/ThemePrefs';
-import {Computed} from 'grainjs';
 import {Rpc} from 'grain-rpc';
 
 /**
@@ -18,7 +16,6 @@ export class DocPluginManager {
   private _clientScope = this._options.clientScope;
   private _docComm = this._options.docComm;
   private _localPlugins = this._options.plugins;
-  private _theme = this._options.theme;
   private _untrustedContentOrigin = this._options.untrustedContentOrigin;
 
   constructor(private _options: {
@@ -26,7 +23,6 @@ export class DocPluginManager {
     untrustedContentOrigin: string,
     docComm: ActiveDocAPI,
     clientScope: ClientScope,
-    theme: Computed<Theme>,
   }) {
     this.pluginsList = [];
     for (const plugin of this._localPlugins) {
@@ -38,7 +34,6 @@ export class DocPluginManager {
           clientScope: this._clientScope,
           untrustedContentOrigin: this._untrustedContentOrigin,
           mainPath: components.safeBrowser,
-          theme: this._theme,
         });
         if (components.safeBrowser) {
           pluginInstance.rpc.registerForwarder(components.safeBrowser, safeBrowser);

app/client/lib/HomePluginManager.ts

@@ -2,8 +2,6 @@ import {ClientScope} from 'app/client/components/ClientScope';
 import {SafeBrowser} from 'app/client/lib/SafeBrowser';
 import {LocalPlugin} from 'app/common/plugin';
 import {createRpcLogger, PluginInstance} from 'app/common/PluginInstance';
-import {Theme} from 'app/common/ThemePrefs';
-import {Computed} from 'grainjs';
 
 /**
  * Home plugins are all plugins that contributes to a general Grist management tasks.
@@ -19,9 +17,8 @@ export class HomePluginManager {
     localPlugins: LocalPlugin[],
     untrustedContentOrigin: string,
     clientScope: ClientScope,
-    theme: Computed<Theme>,
   }) {
-    const {localPlugins, untrustedContentOrigin, clientScope, theme} = options;
+    const {localPlugins, untrustedContentOrigin, clientScope} = options;
     this.pluginsList = [];
     for (const plugin of localPlugins) {
       try {
@@ -41,7 +38,6 @@ export class HomePluginManager {
           clientScope,
           untrustedContentOrigin,
           mainPath: components.safeBrowser,
-          theme,
         });
         if (components.safeBrowser) {
           pluginInstance.rpc.registerForwarder(components.safeBrowser, safeBrowser);

app/client/lib/ReferenceUtils.ts

@@ -1,22 +1,36 @@
+import {ACIndex, ACResults} from 'app/client/lib/ACIndex';
+import {makeT} from 'app/client/lib/localization';
+import {ICellItem} from 'app/client/models/ColumnACIndexes';
+import {ColumnCache} from 'app/client/models/ColumnCache';
 import {DocData} from 'app/client/models/DocData';
 import {ColumnRec} from 'app/client/models/entities/ColumnRec';
 import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec';
 import {TableData} from 'app/client/models/TableData';
 import {getReferencedTableId, isRefListType} from 'app/common/gristTypes';
+import {EmptyRecordView} from 'app/common/PredicateFormula';
 import {BaseFormatter} from 'app/common/ValueFormatter';
+import {Disposable, dom, Observable} from 'grainjs';
+
+const t = makeT('ReferenceUtils');
 
 /**
  * Utilities for common operations involving Ref[List] fields.
  */
-export class ReferenceUtils {
+export class ReferenceUtils extends Disposable {
   public readonly refTableId: string;
   public readonly tableData: TableData;
   public readonly visibleColFormatter: BaseFormatter;
   public readonly visibleColModel: ColumnRec;
   public readonly visibleColId: string;
   public readonly isRefList: boolean;
+  public readonly hasDropdownCondition = Boolean(this.field.dropdownCondition.peek()?.text);
+
+  private readonly _columnCache: ColumnCache<ACIndex<ICellItem>>;
+  private _dropdownConditionError = Observable.create<string | null>(this, null);
+
+  constructor(public readonly field: ViewFieldRec, private readonly _docData: DocData) {
+    super();
 
-  constructor(public readonly field: ViewFieldRec, docData: DocData) {
     const colType = field.column().type();
     const refTableId = getReferencedTableId(colType);
     if (!refTableId) {
@@ -24,7 +38,7 @@ export class ReferenceUtils {
     }
     this.refTableId = refTableId;
 
-    const tableData = docData.getTable(refTableId);
+    const tableData = _docData.getTable(refTableId);
     if (!tableData) {
       throw new Error("Invalid referenced table " + refTableId);
     }
@@ -34,6 +48,8 @@ export class ReferenceUtils {
     this.visibleColModel = field.visibleColModel();
     this.visibleColId = this.visibleColModel.colId() || 'id';
     this.isRefList = isRefListType(colType);
+
+    this._columnCache = new ColumnCache<ACIndex<ICellItem>>(this.tableData);
   }
 
   public idToText(value: unknown) {
@@ -43,10 +59,86 @@ export class ReferenceUtils {
     return String(value || '');
   }
 
-  public autocompleteSearch(text: string) {
+  /**
-    const acIndex = this.tableData.columnACIndexes.getColACIndex(this.visibleColId, this.visibleColFormatter);
+   * Searches the autocomplete index for the given `text`, returning
+   * all matching results and related metadata.
+   *
+   * If a dropdown condition is set, results are dependent on the `rowId`
+   * that the autocomplete dropdown is open in. Otherwise, `rowId` has no
+   * effect.
+   */
+  public autocompleteSearch(text: string, rowId: number): ACResults<ICellItem> {
+    let acIndex: ACIndex<ICellItem>;
+    if (this.hasDropdownCondition) {
+      try {
+        acIndex = this._getDropdownConditionACIndex(rowId);
+      } catch (e) {
+        this._dropdownConditionError?.set(e);
+        return {items: [], extraItems: [], highlightFunc: () => [], selectIndex: -1};
+      }
+    } else {
+      acIndex = this.tableData.columnACIndexes.getColACIndex(
+        this.visibleColId,
+        this.visibleColFormatter,
+      );
+    }
     return acIndex.search(text);
   }
+
+  public buildNoItemsMessage() {
+    return dom.domComputed(use => {
+      const error = use(this._dropdownConditionError);
+      if (error) { return t('Error in dropdown condition'); }
+
+      return this.hasDropdownCondition
+        ? t('No choices matching condition')
+        : t('No choices to select');
+    });
+  }
+
+  /**
+   * Returns a column index for the visible column, filtering the items in the
+   * index according to the set dropdown condition.
+   *
+   * This method is similar to `this.tableData.columnACIndexes.getColACIndex`,
+   * but whereas that method caches indexes globally, this method does so
+   * locally (as a new instances of this class is created each time a Reference
+   * or Reference List editor is created).
+   *
+   * It's important that this method be used when a dropdown condition is set,
+   * as items in indexes that don't satisfy the dropdown condition need to be
+   * filtered.
+   */
+  private _getDropdownConditionACIndex(rowId: number) {
+    return this._columnCache.getValue(
+      this.visibleColId,
+      () => this.tableData.columnACIndexes.buildColACIndex(
+        this.visibleColId,
+        this.visibleColFormatter,
+        this._buildDropdownConditionACFilter(rowId)
+      )
+    );
+  }
+
+  private _buildDropdownConditionACFilter(rowId: number) {
+    const dropdownConditionCompiled = this.field.dropdownConditionCompiled.get();
+    if (dropdownConditionCompiled?.kind !== 'success') {
+      throw new Error('Dropdown condition is not compiled');
+    }
+
+    const tableId = this.field.tableId.peek();
+    const table = this._docData.getTable(tableId);
+    if (!table) { throw new Error(`Table ${tableId} not found`); }
+
+    const {result: predicate} = dropdownConditionCompiled;
+    const rec = table.getRecord(rowId) || new EmptyRecordView();
+    return (item: ICellItem) => {
+      const choice = item.rowId === 'new' ? new EmptyRecordView() : this.tableData.getRecord(item.rowId);
+      if (!choice) { throw new Error(`Reference ${item.rowId} not found`); }
+
+      return predicate({rec, choice});
+    };
+  }
 }
 
 export function nocaseEqual(a: string, b: string) {

app/client/lib/SafeBrowser.ts

@@ -33,6 +33,7 @@ import { ClientScope } from 'app/client/components/ClientScope';
 import { get as getBrowserGlobals } from 'app/client/lib/browserGlobals';
 import dom from 'app/client/lib/dom';
 import * as Mousetrap from 'app/client/lib/Mousetrap';
+import { gristThemeObs } from 'app/client/ui2018/theme';
 import { ActionRouter } from 'app/common/ActionRouter';
 import { BaseComponent, BaseLogger, createRpcLogger, PluginInstance, warnIfNotReady } from 'app/common/PluginInstance';
 import { tbind } from 'app/common/tbind';
@@ -41,7 +42,7 @@ import { getOriginUrl } from 'app/common/urlUtils';
 import { GristAPI, RPC_GRISTAPI_INTERFACE } from 'app/plugin/GristAPI';
 import { RenderOptions, RenderTarget } from 'app/plugin/RenderOptions';
 import { checkers } from 'app/plugin/TypeCheckers';
-import { Computed, dom as grainjsDom, Observable } from 'grainjs';
+import { dom as grainjsDom, Observable } from 'grainjs';
 import { IMsgCustom, IMsgRpcCall, IRpcLogger, MsgType, Rpc } from 'grain-rpc';
 import { Disposable } from './dispose';
 import isEqual from 'lodash/isEqual';
@@ -73,8 +74,6 @@ export class SafeBrowser extends BaseComponent {
       new IframeProcess(safeBrowser, rpc, src);
   }
 
-  public theme = this._options.theme;
-
   // All view processes. This is not used anymore to dispose all processes on deactivation (this is
   // now achieved using `this._mainProcess.autoDispose(...)`) but rather to be able to dispatch
   // events to all processes (such as doc actions which will need soon).
@@ -94,7 +93,6 @@ export class SafeBrowser extends BaseComponent {
     pluginInstance: PluginInstance,
     clientScope: ClientScope,
     untrustedContentOrigin: string,
-    theme: Computed<Theme>,
     mainPath?: string,
     baseLogger?: BaseLogger,
     rpcLogger?: IRpcLogger,
@@ -312,7 +310,7 @@ class IframeProcess extends ViewProcess {
     const listener = async (event: MessageEvent) => {
       if (event.source === iframe.contentWindow) {
         if (event.data.mtype === MsgType.Ready) {
-          await this._sendTheme({theme: safeBrowser.theme.get(), fromReady: true});
+          await this._sendTheme({theme: gristThemeObs().get(), fromReady: true});
         }
 
         if (event.data.data?.message === 'themeInitialized') {
@@ -328,15 +326,11 @@ class IframeProcess extends ViewProcess {
     });
     this.rpc.setSendMessage(msg => iframe.contentWindow!.postMessage(msg, '*'));
 
-    if (safeBrowser.theme) {
+    this.autoDispose(gristThemeObs().addListener(async (newTheme, oldTheme) => {
-      this.autoDispose(
+      if (isEqual(newTheme, oldTheme)) { return; }
-        safeBrowser.theme.addListener(async (newTheme, oldTheme) => {
-          if (isEqual(newTheme, oldTheme)) { return; }
 
-          await this._sendTheme({theme: newTheme});
+      await this._sendTheme({theme: newTheme});
-        })
+    }));
-      );
-    }
   }
 
   private async _sendTheme({theme, fromReady = false}: {theme: Theme, fromReady?: boolean}) {

app/client/lib/autocomplete.ts

@@ -4,7 +4,8 @@
 import {createPopper, Modifier, Instance as Popper, Options as PopperOptions} from '@popperjs/core';
 import {ACItem, ACResults, HighlightFunc} from 'app/client/lib/ACIndex';
 import {reportError} from 'app/client/models/errors';
-import {Disposable, dom, EventCB, IDisposable} from 'grainjs';
+import {testId, theme} from 'app/client/ui2018/cssVars';
+import {Disposable, dom, DomContents, EventCB, IDisposable} from 'grainjs';
 import {obsArray, onKeyElem, styled} from 'grainjs';
 import merge = require('lodash/merge');
 import maxSize from 'popper-max-size-modifier';
@@ -26,6 +27,9 @@ export interface IAutocompleteOptions<Item extends ACItem> {
   // Defaults to the document body.
   attach?: Element|string|null;
 
+  // If provided, builds and shows the message when there are no items (excluding any extra items).
+  buildNoItemsMessage?: () => DomContents;
+
   // Given a search term, return the list of Items to render.
   search(searchText: string): Promise<ACResults<Item>>;
 
@@ -46,7 +50,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
   // The UL element containing the actual menu items.
   protected _menuContent: HTMLElement;
 
-  // Index into _items as well as into _menuContent, -1 if nothing selected.
+  // Index into _menuContent, -1 if nothing selected.
   protected _selectedIndex: number = -1;
 
   // Currently selected element.
@@ -56,6 +60,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
   private _mouseOver: {reset(): void};
   private _lastAsTyped: string;
   private _items = this.autoDispose(obsArray<Item>([]));
+  private _extraItems = this.autoDispose(obsArray<Item>([]));
   private _highlightFunc: HighlightFunc;
 
   constructor(
@@ -65,14 +70,19 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
     super();
 
     const content = cssMenuWrap(
-      this._menuContent = cssMenu({class: _options.menuCssClass || ''},
+      cssMenu(
-        dom.forEach(this._items, (item) => _options.renderItem(item, this._highlightFunc)),
+        {class: _options.menuCssClass || ''},
         dom.style('min-width', _triggerElem.getBoundingClientRect().width + 'px'),
-        dom.on('mouseleave', (ev) => this._setSelected(-1, true)),
+        this._maybeShowNoItemsMessage(),
-        dom.on('click', (ev) => {
+        this._menuContent = dom('div',
-          this._setSelected(this._findTargetItem(ev.target), true);
+          dom.forEach(this._items, (item) => _options.renderItem(item, this._highlightFunc)),
-          if (_options.onClick) { _options.onClick(); }
+          dom.forEach(this._extraItems, (item) => _options.renderItem(item, this._highlightFunc)),
-        })
+          dom.on('mouseleave', (ev) => this._setSelected(-1, true)),
+          dom.on('click', (ev) => {
+            this._setSelected(this._findTargetItem(ev.target), true);
+            if (_options.onClick) { _options.onClick(); }
+          }),
+        ),
       ),
       // Prevent trigger element from being blurred on click.
       dom.on('mousedown', (ev) => ev.preventDefault()),
@@ -104,7 +114,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
   }
 
   public getSelectedItem(): Item|undefined {
-    return this._items.get()[this._selectedIndex];
+    return this._allItems[this._selectedIndex];
   }
 
   public search(findMatch?: (items: Item[]) => number) {
@@ -145,7 +155,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
 
   private _getNext(step: 1 | -1): number {
     // Pretend there is an extra element at the end to mean "nothing selected".
-    const xsize = this._items.get().length + 1;
+    const xsize = this._allItems.length + 1;
     const next = (this._selectedIndex + step + xsize) % xsize;
     return (next === xsize - 1) ? -1 : next;
   }
@@ -157,6 +167,7 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
     const acResults = await this._options.search(inputVal);
     this._highlightFunc = acResults.highlightFunc;
     this._items.set(acResults.items);
+    this._extraItems.set(acResults.extraItems);
 
     // Plain update() (which is deferred) may be better, but if _setSelected() causes scrolling
     // before the positions are updated, it causes the entire page to scroll horizontally.
@@ -166,12 +177,24 @@ export class Autocomplete<Item extends ACItem> extends Disposable {
 
     let index: number;
     if (findMatch) {
-      index = findMatch(this._items.get());
+      index = findMatch(this._allItems);
     } else {
       index = inputVal ? acResults.selectIndex : -1;
     }
     this._setSelected(index, false);
   }
+
+  private get _allItems() {
+    return [...this._items.get(), ...this._extraItems.get()];
+  }
+
+  private _maybeShowNoItemsMessage() {
+    const {buildNoItemsMessage} = this._options;
+    if (!buildNoItemsMessage) { return null; }
+
+    return dom.maybe(use => use(this._items).length === 0, () =>
+      cssNoItemsMessage(buildNoItemsMessage(), testId('autocomplete-no-items-message')));
+  }
 }
 
 
@@ -253,3 +276,10 @@ const cssMenuWrap = styled('div', `
   flex-direction: column;
   outline: none;
 `);
+
+const cssNoItemsMessage = styled('div', `
+  color: ${theme.lightText};
+  padding: var(--weaseljs-menu-item-padding, 8px 24px);
+  text-align: center;
+  user-select: none;
+`);

app/client/lib/imports.d.ts

@@ -6,17 +6,20 @@ import * as GristDocModule from 'app/client/components/GristDoc';
 import * as ViewPane from 'app/client/components/ViewPane';
 import * as UserManagerModule from 'app/client/ui/UserManager';
 import * as searchModule from 'app/client/ui2018/search';
+import * as ace from 'ace-builds';
 import * as momentTimezone from 'moment-timezone';
 import * as plotly from 'plotly.js';
 
-export type PlotlyType = typeof plotly;
+export type Ace = typeof ace;
 export type MomentTimezone = typeof momentTimezone;
+export type PlotlyType = typeof plotly;
 
 export function loadAccountPage(): Promise<typeof AccountPageModule>;
 export function loadActivationPage(): Promise<typeof ActivationPageModule>;
 export function loadBillingPage(): Promise<typeof BillingPageModule>;
 export function loadAdminPanel(): Promise<typeof AdminPanelModule>;
 export function loadGristDoc(): Promise<typeof GristDocModule>;
+export function loadAce(): Promise<Ace>;
 export function loadMomentTimezone(): Promise<MomentTimezone>;
 export function loadPlotly(): Promise<PlotlyType>;
 export function loadSearch(): Promise<typeof searchModule>;

app/client/lib/imports.js

@@ -13,6 +13,17 @@ exports.loadAdminPanel = () => import('app/client/ui/AdminPanel' /* webpackChunk
 exports.loadGristDoc = () => import('app/client/components/GristDoc' /* webpackChunkName: "GristDoc" */);
 // When importing this way, the module is under the "default" member, not sure why (maybe
 // esbuild-loader's doing).
+exports.loadAce = () => import('ace-builds')
+  .then(async (m) => {
+    await Promise.all([
+      import('ace-builds/src-noconflict/ext-static_highlight'),
+      import('ace-builds/src-noconflict/mode-python'),
+      import('ace-builds/src-noconflict/theme-chrome'),
+      import('ace-builds/src-noconflict/theme-dracula'),
+    ]);
+
+    return m.default;
+  });
 exports.loadMomentTimezone = () => import('moment-timezone').then(m => m.default);
 exports.loadPlotly = () => import('plotly.js-basic-dist' /* webpackChunkName: "plotly" */);
 exports.loadSearch = () => import('app/client/ui2018/search' /* webpackChunkName: "search" */);

app/client/lib/nameUtils.ts

@@ -0,0 +1,14 @@
+/**
+ * We allow alphanumeric characters and certain common whitelisted characters (except at the start),
+ * plus everything non-ASCII (for non-English alphabets, which we want to allow but it's hard to be
+ * more precise about what exactly to allow).
+ */
+// eslint-disable-next-line no-control-regex
+const VALID_NAME_REGEXP = /^(\w|[^\u0000-\u007F])(\w|[- ./'"()]|[^\u0000-\u007F])*$/;
+
+/**
+ * Test name against various rules to check if it is a valid username.
+ */
+export function checkName(name: string): boolean {
+  return VALID_NAME_REGEXP.test(name);
+}

app/client/lib/timeUtils.ts

@@ -0,0 +1,21 @@
+import moment from 'moment';
+
+/**
+ * Given a UTC Date ISO 8601 string (the doc updatedAt string), gives a reader-friendly
+ * relative time to now - e.g. 'yesterday', '2 days ago'.
+ */
+export function getTimeFromNow(utcDateISO: string): string {
+  const time = moment.utc(utcDateISO);
+  const now = moment();
+  const diff = now.diff(time, 's');
+  if (diff < 0 && diff > -60) {
+    // If the time appears to be in the future, but less than a minute
+    // in the future, chalk it up to a difference in time
+    // synchronization and don't claim the resource will be changed in
+    // the future.  For larger differences, just report them
+    // literally, there's a more serious problem or lack of
+    // synchronization.
+    return now.fromNow();
+  }
+  return time.fromNow();
+}

app/client/models/AppModel.ts

@@ -10,7 +10,7 @@ import {Notifier} from 'app/client/models/NotifyModel';
 import {getFlavor, ProductFlavor} from 'app/client/ui/CustomThemes';
 import {buildNewSiteModal, buildUpgradeModal} from 'app/client/ui/ProductUpgrades';
 import {SupportGristNudge} from 'app/client/ui/SupportGristNudge';
-import {prefersDarkModeObs} from 'app/client/ui2018/cssVars';
+import {gristThemePrefs} from 'app/client/ui2018/theme';
 import {AsyncCreate} from 'app/common/AsyncCreate';
 import {ICustomWidget} from 'app/common/CustomWidget';
 import {OrgUsageSummary} from 'app/common/DocUsage';
@@ -21,9 +21,7 @@ import {LocalPlugin} from 'app/common/plugin';
 import {DismissedPopup, DismissedReminder, UserPrefs} from 'app/common/Prefs';
 import {isOwner, isOwnerOrEditor} from 'app/common/roles';
 import {getTagManagerScript} from 'app/common/tagManager';
-import {getDefaultThemePrefs, Theme, ThemeColors, ThemePrefs,
+import {getDefaultThemePrefs, ThemePrefs, ThemePrefsChecker} from 'app/common/ThemePrefs';
-        ThemePrefsChecker} from 'app/common/ThemePrefs';
-import {getThemeColors} from 'app/common/Themes';
 import {getGristConfig} from 'app/common/urlUtils';
 import {ExtendedUser} from 'app/common/UserAPI';
 import {getOrgName, isTemplatesOrg, Organization, OrgError, UserAPI, UserAPIImpl} from 'app/common/UserAPI';
@@ -118,7 +116,6 @@ export interface AppModel {
 
   userPrefsObs: Observable<UserPrefs>;
   themePrefs: Observable<ThemePrefs>;
-  currentTheme: Computed<Theme>;
   /**
    * Popups that user has seen.
    */
@@ -170,8 +167,9 @@ export class TopAppModelImpl extends Disposable implements TopAppModel {
   private readonly _widgets: AsyncCreate<ICustomWidget[]>;
 
   constructor(window: {gristConfig?: GristLoadConfig},
-              public readonly api: UserAPI = newUserAPIImpl(),
+    public readonly api: UserAPI = newUserAPIImpl(),
-              public readonly options: TopAppModelOptions = {}) {
+    public readonly options: TopAppModelOptions = {}
+  ) {
     super();
     setErrorNotifier(this.notifier);
     this.isSingleOrg = Boolean(window.gristConfig && window.gristConfig.singleOrg);
@@ -307,7 +305,6 @@ export class AppModelImpl extends Disposable implements AppModel {
     defaultValue: getDefaultThemePrefs(),
     checker: ThemePrefsChecker,
   }) as Observable<ThemePrefs>;
-  public readonly currentTheme = this._getCurrentThemeObs();
 
   public readonly dismissedPopups = getUserPrefObs(this.userPrefsObs, 'dismissedPopups',
     { defaultValue: [] }) as Observable<DismissedPopup[]>;
@@ -359,6 +356,11 @@ export class AppModelImpl extends Disposable implements AppModel {
     public readonly orgError?: OrgError,
   ) {
     super();
+
+    // Whenever theme preferences change, update the global `gristThemePrefs` observable; this triggers
+    // an automatic update to the global `gristThemeObs` computed observable.
+    this.autoDispose(subscribe(this.themePrefs, (_use, themePrefs) => gristThemePrefs.set(themePrefs)));
+
     this._recordSignUpIfIsNewUser();
 
     const state = urlState().state.get();
@@ -493,41 +495,14 @@ export class AppModelImpl extends Disposable implements AppModel {
     dataLayer.push({event: 'new-sign-up'});
     getUserPrefObs(this.userPrefsObs, 'recordSignUpEvent').set(undefined);
   }
+}
 
-  private _getCurrentThemeObs() {
+export function getOrgNameOrGuest(org: Organization|null, user: FullUser|null) {
-    return Computed.create(this, this.themePrefs, prefersDarkModeObs(),
+  if (!org) { return ''; }
-      (_use, themePrefs, prefersDarkMode) => {
+  if (user && user.anonymous && org.owner && org.owner.id === user.id) {
-        let {appearance, syncWithOS} = themePrefs;
+    return "@Guest";
-
-        const urlParams = urlState().state.get().params;
-        if (urlParams?.themeAppearance) {
-          appearance = urlParams?.themeAppearance;
-        }
-
-        if (urlParams?.themeSyncWithOs !== undefined) {
-          syncWithOS = urlParams?.themeSyncWithOs;
-        }
-
-        if (syncWithOS) {
-          appearance = prefersDarkMode ? 'dark' : 'light';
-        }
-
-        let nameOrColors = themePrefs.colors[appearance];
-        if (urlParams?.themeName) {
-          nameOrColors = urlParams?.themeName;
-        }
-
-        let colors: ThemeColors;
-        if (typeof nameOrColors === 'string') {
-          colors = getThemeColors(nameOrColors);
-        } else {
-          colors = nameOrColors;
-        }
-
-        return {appearance, colors};
-      },
-    );
   }
+  return getOrgName(org);
 }
 
 export function getHomeUrl(): string {
@@ -541,11 +516,3 @@ export function newUserAPIImpl(): UserAPIImpl {
     fetch: hooks.fetch,
   });
 }
-
-export function getOrgNameOrGuest(org: Organization|null, user: FullUser|null) {
-  if (!org) { return ''; }
-  if (user && user.anonymous && org.owner && org.owner.id === user.id) {
-    return "@Guest";
-  }
-  return getOrgName(org);
-}

app/client/models/ColumnACIndexes.ts

@@ -21,7 +21,6 @@ export interface ICellItem {
   cleanText: string;      // Trimmed lowercase text for searching.
 }
 
-
 export class ColumnACIndexes {
   private _columnCache = new ColumnCache<ACIndex<ICellItem>>(this._tableData);
 
@@ -33,22 +32,28 @@ export class ColumnACIndexes {
    * getColACIndex() is called for the same column with the the same formatter.
    */
   public getColACIndex(colId: string, formatter: BaseFormatter): ACIndex<ICellItem> {
-    return this._columnCache.getValue(colId, () => this._buildColACIndex(colId, formatter));
+    return this._columnCache.getValue(colId, () => this.buildColACIndex(colId, formatter));
   }
 
-  private _buildColACIndex(colId: string, formatter: BaseFormatter): ACIndex<ICellItem> {
+  public buildColACIndex(
+    colId: string,
+    formatter: BaseFormatter,
+    filter?: (item: ICellItem) => boolean
+  ): ACIndex<ICellItem> {
     const rowIds = this._tableData.getRowIds();
     const valColumn = this._tableData.getColValues(colId);
     if (!valColumn) {
       throw new UserError(`Invalid column ${this._tableData.tableId}.${colId}`);
     }
-    const items: ICellItem[] = valColumn.map((val, i) => {
+    const items: ICellItem[] = valColumn
-      const rowId = rowIds[i];
+      .map((val, i) => {
-      const text = formatter.formatAny(val);
+        const rowId = rowIds[i];
-      const cleanText = normalizeText(text);
+        const text = formatter.formatAny(val);
-      return {rowId, text, cleanText};
+        const cleanText = normalizeText(text);
-    });
+        return {rowId, text, cleanText};
-    items.sort(itemCompare);
+      })
+      .filter((item) => filter?.(item) ?? true)
+      .sort(itemCompare);
     return new ACIndexImpl(items);
   }
 }

app/client/models/HomeModel.ts

@@ -14,30 +14,11 @@ import * as roles from 'app/common/roles';
 import {getGristConfig} from 'app/common/urlUtils';
 import {Document, Organization, Workspace} from 'app/common/UserAPI';
 import {bundleChanges, Computed, Disposable, Observable, subscribe} from 'grainjs';
-import moment from 'moment';
 import flatten = require('lodash/flatten');
 import sortBy = require('lodash/sortBy');
 
 const DELAY_BEFORE_SPINNER_MS = 500;
 
-// Given a UTC Date ISO 8601 string (the doc updatedAt string), gives a reader-friendly
-// relative time to now - e.g. 'yesterday', '2 days ago'.
-export function getTimeFromNow(utcDateISO: string): string {
-  const time = moment.utc(utcDateISO);
-  const now = moment();
-  const diff = now.diff(time, 's');
-  if (diff < 0 && diff > -60) {
-    // If the time appears to be in the future, but less than a minute
-    // in the future, chalk it up to a difference in time
-    // synchronization and don't claim the resource will be changed in
-    // the future.  For larger differences, just report them
-    // literally, there's a more serious problem or lack of
-    // synchronization.
-    return now.fromNow();
-  }
-  return time.fromNow();
-}
-
 export interface HomeModel {
   // PageType value, one of the discriminated union values used by AppModel.
   pageType: "home";
@@ -190,7 +171,6 @@ export class HomeModelImpl extends Disposable implements HomeModel, ViewSettings
       localPlugins: _app.topAppModel.plugins,
       untrustedContentOrigin: _app.topAppModel.getUntrustedContentOrigin()!,
       clientScope,
-      theme: _app.currentTheme,
     });
     const importSources = ImportSourceElement.fromArray(pluginManager.pluginsList);
     this.importSources.set(importSources);

app/client/models/entities/ViewFieldRec.ts

@@ -2,12 +2,15 @@ import {ColumnRec, DocModel, IRowModel, refListRecords, refRecord, ViewSectionRe
 import {formatterForRec} from 'app/client/models/entities/ColumnRec';
 import * as modelUtil from 'app/client/models/modelUtil';
 import {removeRule, RuleOwner} from 'app/client/models/RuleOwner';
-import { HeaderStyle, Style } from 'app/client/models/Styles';
+import {HeaderStyle, Style} from 'app/client/models/Styles';
 import {ViewFieldConfig} from 'app/client/models/ViewFieldConfig';
 import * as UserType from 'app/client/widgets/UserType';
 import {DocumentSettings} from 'app/common/DocumentSettings';
+import {DropdownCondition, DropdownConditionCompilationResult} from 'app/common/DropdownCondition';
+import {compilePredicateFormula} from 'app/common/PredicateFormula';
 import {BaseFormatter} from 'app/common/ValueFormatter';
 import {createParser} from 'app/common/ValueParser';
+import {Computed} from 'grainjs';
 import * as ko from 'knockout';
 
 // Represents a page entry in the tree of pages.
@@ -106,6 +109,9 @@ export interface ViewFieldRec extends IRowModel<"_grist_Views_section_field">, R
   /** Label in FormView. By default FormView uses label, use this to override it. */
   question: modelUtil.KoSaveableObservable<string|undefined>;
 
+  dropdownCondition: modelUtil.KoSaveableObservable<DropdownCondition|undefined>;
+  dropdownConditionCompiled: Computed<DropdownConditionCompilationResult|null>;
+
   createValueParser(): (value: string) => any;
 
   // Helper which adds/removes/updates field's displayCol to match the formula.
@@ -316,4 +322,21 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void
 
   this.disableModify = this.autoDispose(ko.pureComputed(() => this.column().disableModify()));
   this.disableEditData = this.autoDispose(ko.pureComputed(() => this.column().disableEditData()));
+
+  this.dropdownCondition = this.widgetOptionsJson.prop('dropdownCondition');
+  this.dropdownConditionCompiled = Computed.create(this, use => {
+    const dropdownCondition = use(this.dropdownCondition);
+    if (!dropdownCondition?.parsed) { return null; }
+
+    try {
+      return {
+        kind: 'success',
+        result: compilePredicateFormula(JSON.parse(dropdownCondition.parsed), {
+          variant: 'dropdown-condition',
+        }),
+      };
+    } catch (e) {
+      return {kind: 'failure', error: e.message};
+    }
+  });
 }

app/client/ui/AccountPage.ts

@@ -1,4 +1,5 @@
 import {detectCurrentLang, makeT} from 'app/client/lib/localization';
+import {checkName} from 'app/client/lib/nameUtils';
 import {AppModel, reportError} from 'app/client/models/AppModel';
 import {urlState} from 'app/client/models/gristUrlState';
 import * as css from 'app/client/ui/AccountPageCss';
@@ -249,23 +250,6 @@ designed to ensure that you're the only person who can access your account, even
   }
 }
 
-/**
- * We allow alphanumeric characters and certain common whitelisted characters (except at the start),
- * plus everything non-ASCII (for non-English alphabets, which we want to allow but it's hard to be
- * more precise about what exactly to allow).
- */
-// eslint-disable-next-line no-control-regex
-const VALID_NAME_REGEXP = /^(\w|[^\u0000-\u007F])(\w|[- ./'"()]|[^\u0000-\u007F])*$/;
-
-/**
- * Test name against various rules to check if it is a valid username.
- */
-export function checkName(name: string): boolean {
-  return VALID_NAME_REGEXP.test(name);
-}
-
-
-
 const cssWarnings = styled(css.warning, `
   margin: -8px 0 0 110px;
 `);

app/client/ui/App.ts

@@ -14,6 +14,7 @@ import {setUpErrorHandling} from 'app/client/models/errors';
 import {createAppUI} from 'app/client/ui/AppUI';
 import {addViewportTag} from 'app/client/ui/viewport';
 import {attachCssRootVars} from 'app/client/ui2018/cssVars';
+import {attachTheme} from 'app/client/ui2018/theme';
 import {BaseAPI} from 'app/common/BaseAPI';
 import {CommDocError} from 'app/common/CommTypes';
 import {DisposableWithEvents} from 'app/common/DisposableWithEvents';
@@ -183,6 +184,7 @@ export class App extends DisposableWithEvents {
 
     // Add the cssRootVars class to enable the variables in cssVars.
     attachCssRootVars(this.topAppModel.productFlavor);
+    attachTheme();
     addViewportTag();
     this.autoDispose(createAppUI(this.topAppModel, this));
   }

app/client/ui/AppUI.ts

@@ -17,7 +17,7 @@ import {pagePanels} from 'app/client/ui/PagePanels';
 import {RightPanel} from 'app/client/ui/RightPanel';
 import {createTopBarDoc, createTopBarHome} from 'app/client/ui/TopBar';
 import {WelcomePage} from 'app/client/ui/WelcomePage';
-import {attachTheme, testId} from 'app/client/ui2018/cssVars';
+import {testId} from 'app/client/ui2018/cssVars';
 import {getPageTitleSuffix} from 'app/common/gristUrls';
 import {getGristConfig} from 'app/common/urlUtils';
 import {Computed, dom, IDisposable, IDisposableOwner, Observable, replaceContent, subscribe} from 'grainjs';
@@ -27,9 +27,7 @@ import {Computed, dom, IDisposable, IDisposableOwner, Observable, replaceContent
 // TODO once #newui is gone, we don't need to worry about this being disposable.
 // appObj is the App object from app/client/ui/App.ts
 export function createAppUI(topAppModel: TopAppModel, appObj: App): IDisposable {
-  const content = dom.maybeOwned(topAppModel.appObs, (owner, appModel) => {
+  const content = dom.maybe(topAppModel.appObs, (appModel) => {
-    owner.autoDispose(attachTheme(appModel.currentTheme));
-
     return [
       createMainPage(appModel, appObj),
       buildSnackbarDom(appModel.notifier, appModel),

app/client/ui/CodeHighlight.ts

@@ -1,45 +1,112 @@
-import * as ace from 'ace-builds';
+import {Ace, loadAce} from 'app/client/lib/imports';
 import {theme, vars} from 'app/client/ui2018/cssVars';
-import {Theme} from 'app/common/ThemePrefs';
+import {gristThemeObs} from 'app/client/ui2018/theme';
-import {getGristConfig} from 'app/common/urlUtils';
+import {
-import {BindableValue, Computed, dom, DomElementArg, Observable, styled, subscribeElem} from 'grainjs';
+  BindableValue,
+  Disposable,
+  DomElementArg,
+  Observable,
+  styled,
+  subscribeElem,
+} from 'grainjs';
 
-// ace-builds also has a minified build (src-min-noconflict), but we don't
+interface BuildCodeHighlighterOptions {
-// use it since webpack already handles minification.
-require('ace-builds/src-noconflict/ext-static_highlight');
-require('ace-builds/src-noconflict/mode-python');
-require('ace-builds/src-noconflict/theme-chrome');
-require('ace-builds/src-noconflict/theme-dracula');
-
-export interface ICodeOptions {
-  gristTheme: Computed<Theme>;
-  placeholder?: string;
   maxLines?: number;
 }
 
+let _ace: Ace;
+let _highlighter: any;
+let _PythonMode: any;
+let _aceDom: any;
+let _chrome: any;
+let _dracula: any;
+let _mode: any;
+
+async function fetchAceModules() {
+  return {
+    ace: _ace || (_ace = await loadAce()),
+    highlighter: _highlighter || (_highlighter = _ace.require('ace/ext/static_highlight')),
+    PythonMode: _PythonMode || (_PythonMode = _ace.require('ace/mode/python').Mode),
+    aceDom: _aceDom || (_aceDom = _ace.require('ace/lib/dom')),
+    chrome: _chrome || (_chrome = _ace.require('ace/theme/chrome')),
+    dracula: _dracula || (_dracula = _ace.require('ace/theme/dracula')),
+    mode: _mode || (_mode = new _PythonMode()),
+  };
+}
+
+/**
+ * Returns a function that accepts a string of text representing code and returns
+ * a highlighted version of it as an HTML string.
+ *
+ * This is useful for scenarios where highlighted code needs to be displayed outside of
+ * grainjs. For example, when using `marked`'s `highlight` option to highlight code
+ * blocks in a Markdown string.
+ */
+export async function buildCodeHighlighter(options: BuildCodeHighlighterOptions = {}) {
+  const {maxLines} = options;
+  const {highlighter, aceDom, chrome, dracula, mode} = await fetchAceModules();
+
+  return (code: string) => {
+    if (maxLines) {
+      // If requested, trim to maxLines, and add an ellipsis at the end.
+      // (Long lines are also truncated with an ellpsis via text-overflow style.)
+      const lines = code.split(/\n/);
+      if (lines.length > maxLines) {
+        code = lines.slice(0, maxLines).join("\n") + " \u2026";  // Ellipsis
+      }
+    }
+
+    let aceThemeName: 'chrome' | 'dracula';
+    let aceTheme: any;
+    if (gristThemeObs().get().appearance === 'dark') {
+      aceThemeName = 'dracula';
+      aceTheme = dracula;
+    } else {
+      aceThemeName = 'chrome';
+      aceTheme = chrome;
+    }
+
+    // Rendering highlighted code gives you back the HTML to insert into the DOM, as well
+    // as the CSS styles needed to apply the theme. The latter typically isn't included in
+    // the document until an Ace editor is opened, so we explicitly import it here to avoid
+    // leaving highlighted code blocks without a theme applied.
+    const {html, css} = highlighter.render(code, mode, aceTheme, 1, true);
+    aceDom.importCssString(css, `${aceThemeName}-highlighted-code`);
+    return html;
+  };
+}
+
+interface BuildHighlightedCodeOptions extends BuildCodeHighlighterOptions {
+  placeholder?: string;
+}
+
+/**
+ * Builds a block of highlighted `code`.
+ *
+ * Highlighting applies an appropriate Ace theme (Chrome or Dracula) based on
+ * the current Grist theme, and automatically re-applies it whenever the Grist
+ * theme changes.
+ */
 export function buildHighlightedCode(
-  code: BindableValue<string>, options: ICodeOptions, ...args: DomElementArg[]
+  owner: Disposable,
+  code: BindableValue<string>,
+  options: BuildHighlightedCodeOptions,
+  ...args: DomElementArg[]
 ): HTMLElement {
-  const {gristTheme, placeholder, maxLines} = options;
+  const {placeholder, maxLines} = options;
-  const {enableCustomCss} = getGristConfig();
+  const codeText = Observable.create(owner, '');
+  const codeTheme = Observable.create(owner, gristThemeObs().get());
 
-  const highlighter = ace.require('ace/ext/static_highlight');
+  async function updateHighlightedCode(elem: HTMLElement) {
-  const PythonMode = ace.require('ace/mode/python').Mode;
-  const aceDom = ace.require('ace/lib/dom');
-  const chrome = ace.require('ace/theme/chrome');
-  const dracula = ace.require('ace/theme/dracula');
-  const mode = new PythonMode();
-
-  const codeText = Observable.create(null, '');
-  const codeTheme = Observable.create(null, gristTheme.get());
-
-  function updateHighlightedCode(elem: HTMLElement) {
     let text = codeText.get();
     if (!text) {
       elem.textContent = placeholder || '';
       return;
     }
 
+    const {highlighter, aceDom, chrome, dracula, mode} = await fetchAceModules();
+    if (owner.isDisposed()) { return; }
+
     if (maxLines) {
       // If requested, trim to maxLines, and add an ellipsis at the end.
       // (Long lines are also truncated with an ellpsis via text-overflow style.)
@@ -51,7 +118,7 @@ export function buildHighlightedCode(
 
     let aceThemeName: 'chrome' | 'dracula';
     let aceTheme: any;
-    if (codeTheme.get().appearance === 'dark' && !enableCustomCss) {
+    if (codeTheme.get().appearance === 'dark') {
       aceThemeName = 'dracula';
       aceTheme = dracula;
     } else {
@@ -69,15 +136,13 @@ export function buildHighlightedCode(
   }
 
   return cssHighlightedCode(
-    dom.autoDispose(codeText),
+    elem => subscribeElem(elem, code, async (newCodeText) => {
-    dom.autoDispose(codeTheme),
-    elem => subscribeElem(elem, code, (newCodeText) => {
       codeText.set(newCodeText);
-      updateHighlightedCode(elem);
+      await updateHighlightedCode(elem);
       }),
-    elem => subscribeElem(elem, gristTheme, (newCodeTheme) => {
+    elem => subscribeElem(elem, gristThemeObs(), async (newCodeTheme) => {
       codeTheme.set(newCodeTheme);
-      updateHighlightedCode(elem);
+      await updateHighlightedCode(elem);
     }),
     ...args,
   );
@@ -95,9 +160,7 @@ export const cssCodeBlock = styled('div', `
 
 const cssHighlightedCode = styled(cssCodeBlock, `
   position: relative;
-  white-space: pre;
   overflow: hidden;
-  text-overflow: ellipsis;
   border: 1px solid ${theme.highlightedCodeBorder};
   border-radius: 3px;
   min-height: 28px;
@@ -110,20 +173,6 @@ const cssHighlightedCode = styled(cssCodeBlock, `
   & .ace_line {
     overflow: hidden;
     text-overflow: ellipsis;
-  }
+    white-space: nowrap;
-`);
-
-export const cssFieldFormula = styled(buildHighlightedCode, `
-  flex: auto;
-  cursor: pointer;
-  margin-top: 4px;
-  padding-left: 24px;
-  --icon-color: ${theme.accentIcon};
-
-  &-disabled-icon.formula_field_sidepane::before {
-    --icon-color: ${theme.iconDisabled};
-  }
-  &-disabled {
-    pointer-events: none;
   }
 `);

app/client/ui/DocHistory.ts

@@ -1,9 +1,9 @@
 import {makeT} from 'app/client/lib/localization';
 import {createSessionObs} from 'app/client/lib/sessionObs';
+import {getTimeFromNow} from 'app/client/lib/timeUtils';
 import {DocPageModel} from 'app/client/models/DocPageModel';
 import {reportError} from 'app/client/models/errors';
 import {urlState} from 'app/client/models/gristUrlState';
-import {getTimeFromNow} from 'app/client/models/HomeModel';
 import {buildConfigContainer} from 'app/client/ui/RightPanel';
 import {buttonSelect} from 'app/client/ui2018/buttonSelect';
 import {testId, theme, vars} from 'app/client/ui2018/cssVars';

app/client/ui/DocMenu.ts

@@ -4,9 +4,10 @@
  * Orgs, workspaces and docs are fetched asynchronously on build via the passed in API.
  */
 import {loadUserManager} from 'app/client/lib/imports';
+import {getTimeFromNow} from 'app/client/lib/timeUtils';
 import {reportError} from 'app/client/models/AppModel';
 import {docUrl, urlState} from 'app/client/models/gristUrlState';
-import {getTimeFromNow, HomeModel, makeLocalViewSettings, ViewSettings} from 'app/client/models/HomeModel';
+import {HomeModel, makeLocalViewSettings, ViewSettings} from 'app/client/models/HomeModel';
 import {getWorkspaceInfo, workspaceName} from 'app/client/models/WorkspaceInfo';
 import {attachAddNewTip} from 'app/client/ui/AddNewTip';
 import * as css from 'app/client/ui/DocMenuCss';

app/client/ui/FieldConfig.ts

@@ -2,7 +2,7 @@ import {makeT} from 'app/client/lib/localization';
 import {GristDoc} from 'app/client/components/GristDoc';
 import {BEHAVIOR, ColumnRec} from 'app/client/models/entities/ColumnRec';
 import {buildHighlightedCode, cssCodeBlock} from 'app/client/ui/CodeHighlight';
-import {cssBlockedCursor, cssLabel, cssRow} from 'app/client/ui/RightPanelStyles';
+import {cssBlockedCursor, cssFieldFormula, cssLabel, cssRow} from 'app/client/ui/RightPanelStyles';
 import {withInfoTooltip} from 'app/client/ui/tooltips';
 import {buildFormulaTriggers} from 'app/client/ui/TriggerFormulas';
 import {textButton} from 'app/client/ui2018/buttons';
@@ -13,7 +13,6 @@ import {IconName} from 'app/client/ui2018/IconList';
 import {selectMenu, selectOption, selectTitle} from 'app/client/ui2018/menus';
 import {createFormulaErrorObs, cssError} from 'app/client/widgets/FormulaEditor';
 import {sanitizeIdent} from 'app/common/gutil';
-import {Theme} from 'app/common/ThemePrefs';
 import {CursorPos} from 'app/plugin/GristAPI';
 import {bundleChanges, Computed, dom, DomContents, DomElementArg, fromKo, MultiHolder,
         Observable, styled} from 'grainjs';
@@ -139,6 +138,8 @@ export function buildFormulaConfig(
   // And close it dispose it when user opens up behavior menu.
   let formulaField: HTMLElement|null = null;
 
+  const focusFormulaField = () => setTimeout(() => formulaField?.focus(), 0);
+
   // Helper function to clear temporary state (will be called when column changes or formula editor closes)
   const clearState = () => bundleChanges(() => {
     // For a detached editor, we may have already been disposed when user switched page.
@@ -242,7 +243,7 @@ export function buildFormulaConfig(
 
   // Converts data column to formula column.
   const convertDataColumnToFormulaOption = () => selectOption(
-    () => (maybeFormula.set(true), formulaField?.focus()),
+    () => (maybeFormula.set(true), focusFormulaField()),
     t("Clear and make into formula"), 'Script');
 
   // Converts to empty column and opens up the editor. (label is the same, but this is used when we have no formula)
@@ -270,15 +271,15 @@ export function buildFormulaConfig(
   const convertDataColumnToTriggerColumn = () => {
     maybeTrigger.set(true);
     // Open the formula editor.
-    formulaField?.focus();
+    focusFormulaField();
   };
 
   // Converts formula column to trigger formula column.
   const convertFormulaToTrigger = () =>
     gristDoc.convertIsFormula([origColumn.id.peek()], {toFormula: false, noRecalc: false});
 
-  const setFormula = () => (maybeFormula.set(true), formulaField?.focus());
+  const setFormula = () => { maybeFormula.set(true); focusFormulaField(); };
-  const setTrigger = () => (maybeTrigger.set(true), formulaField?.focus());
+  const setTrigger = () => { maybeTrigger.set(true); focusFormulaField(); };
 
   // Actions on save formula. Those actions are using column that comes from FormulaEditor.
   // Formula editor scope is broader then RightPanel, it can be disposed after RightPanel is closed,
@@ -325,16 +326,19 @@ export function buildFormulaConfig(
   const errorMessage = createFormulaErrorObs(owner, gristDoc, origColumn);
   // Helper that will create different flavors for formula builder.
   const formulaBuilder = (onSave: SaveHandler, canDetach?: boolean) => [
-    cssRow(formulaField = buildFormula(
+    cssRow(
-      origColumn,
+      buildFormula(
-      buildEditor,
+        origColumn,
-      {
+        buildEditor,
-        gristTheme: gristDoc.currentTheme,
+        {
-        disabled: disableOtherActions,
+          disabled: disableOtherActions,
-        canDetach,
+          canDetach,
-        onSave,
+          onSave,
-        onCancel: clearState,
+          onCancel: clearState,
-      })),
+        },
+        (el) => { formulaField = el; },
+      )
+    ),
     dom.maybe(errorMessage, errMsg => cssRow(cssError(errMsg), testId('field-error-count'))),
   ];
 
@@ -419,7 +423,6 @@ export function buildFormulaConfig(
 }
 
 interface BuildFormulaOptions {
-  gristTheme: Computed<Theme>;
   disabled: Observable<boolean>;
   canDetach?: boolean;
   onSave?: SaveHandler;
@@ -429,10 +432,12 @@ interface BuildFormulaOptions {
 function buildFormula(
   column: ColumnRec,
   buildEditor: BuildEditor,
-  options: BuildFormulaOptions
+  options: BuildFormulaOptions,
+  ...args: DomElementArg[]
 ) {
-  const {gristTheme, disabled, canDetach = true, onSave, onCancel} = options;
+  const {disabled, canDetach = true, onSave, onCancel} = options;
-  return cssFieldFormula(column.formula, {gristTheme, maxLines: 2},
+  return dom.create(buildHighlightedCode, column.formula, {maxLines: 2},
+    dom.cls(cssFieldFormula.className),
     dom.cls('formula_field_sidepane'),
     cssFieldFormula.cls('-disabled', disabled),
     cssFieldFormula.cls('-disabled-icon', use => !use(column.formula)),
@@ -447,24 +452,10 @@ function buildFormula(
       onSave,
       onCancel,
     })),
+    ...args,
   );
 }
 
-export const cssFieldFormula = styled(buildHighlightedCode, `
-  flex: auto;
-  cursor: pointer;
-  margin-top: 4px;
-  padding-left: 24px;
-  --icon-color: ${theme.accentIcon};
-
-  &-disabled-icon.formula_field_sidepane::before {
-    --icon-color: ${theme.lightText};
-  }
-  &-disabled {
-    pointer-events: none;
-  }
-`);
-
 const cssToggleButton = styled(cssIconButton, `
   margin-left: 8px;
   background-color: ${theme.rightPanelToggleButtonDisabledBg};

app/client/ui/GristTooltips.ts

@@ -1,5 +1,6 @@
 import * as commands from 'app/client/components/commands';
 import {makeT} from 'app/client/lib/localization';
+import {buildHighlightedCode} from 'app/client/ui/CodeHighlight';
 import {ShortcutKey, ShortcutKeyContent} from 'app/client/ui/ShortcutKey';
 import {icon} from 'app/client/ui2018/icons';
 import {cssLink} from 'app/client/ui2018/links';
@@ -39,7 +40,9 @@ export type Tooltip =
   | 'uuid'
   | 'lookups'
   | 'formulaColumn'
-  | 'accessRulesTableWide';
+  | 'accessRulesTableWide'
+  | 'setChoiceDropdownCondition'
+  | 'setRefDropdownCondition';
 
 export type TooltipContentFunc = (...domArgs: DomElementArg[]) => DomContents;
 
@@ -125,7 +128,26 @@ see or edit which parts of your document.')
     ...args,
   ),
   accessRulesTableWide: (...args: DomElementArg[]) => cssTooltipContent(
-    dom('div', t('These rules are applied after all column rules have been processed, if applicable.'))
+    dom('div', t('These rules are applied after all column rules have been processed, if applicable.')),
+    ...args,
+  ),
+  setChoiceDropdownCondition: (...args: DomElementArg[]) => cssTooltipContent(
+    dom('div',
+      t('Filter displayed dropdown values with a condition.')
+    ),
+    dom('div', {style: 'margin-top: 8px;'}, t('Example: {{example}}', {
+      example: dom.create(buildHighlightedCode, 'choice not in $Categories', {}, {style: 'margin-top: 8px;'}),
+    })),
+    ...args,
+  ),
+  setRefDropdownCondition: (...args: DomElementArg[]) => cssTooltipContent(
+    dom('div',
+      t('Filter displayed dropdown values with a condition.')
+    ),
+    dom('div', {style: 'margin-top: 8px;'}, t('Example: {{example}}', {
+      example: dom.create(buildHighlightedCode, 'choice.Role == "Manager"', {}, {style: 'margin-top: 8px;'}),
+    })),
+    ...args,
   ),
 };
 

app/client/ui/PinnedDocs.ts

@@ -1,5 +1,6 @@
+import {getTimeFromNow} from 'app/client/lib/timeUtils';
 import {docUrl, urlState} from 'app/client/models/gristUrlState';
-import {getTimeFromNow, HomeModel} from 'app/client/models/HomeModel';
+import {HomeModel} from 'app/client/models/HomeModel';
 import {makeDocOptionsMenu, makeRemovedDocOptionsMenu} from 'app/client/ui/DocMenu';
 import {transientInput} from 'app/client/ui/transientInput';
 import {colors, theme, vars} from 'app/client/ui2018/cssVars';

app/client/ui/RightPanelStyles.ts

@@ -94,3 +94,18 @@ export const cssPinButton = styled('div', `
 export const cssNumericSpinner = styled(numericSpinner, `
   height: 28px;
 `);
+
+export const cssFieldFormula = styled('div', `
+  flex: auto;
+  cursor: pointer;
+  margin-top: 4px;
+  padding-left: 24px;
+  --icon-color: ${theme.accentIcon};
+
+  &-disabled-icon.formula_field_sidepane::before {
+    --icon-color: ${theme.iconDisabled};
+  }
+  &-disabled {
+    pointer-events: none;
+  }
+`);

app/client/ui/ThemeConfig.ts

@@ -2,7 +2,7 @@ import {makeT} from 'app/client/lib/localization';
 import {AppModel} from 'app/client/models/AppModel';
 import * as css from 'app/client/ui/AccountPageCss';
 import {labeledSquareCheckbox} from 'app/client/ui2018/checkbox';
-import {prefersDarkModeObs} from 'app/client/ui2018/cssVars';
+import {prefersColorSchemeDarkObs} from 'app/client/ui2018/theme';
 import {select} from 'app/client/ui2018/menus';
 import {ThemeAppearance} from 'app/common/ThemePrefs';
 import {Computed, Disposable, dom, makeTestId, styled} from 'grainjs';
@@ -20,10 +20,10 @@ export class ThemeConfig extends Disposable {
   private _appearance = Computed.create(this,
     this._themePrefs,
     this._syncWithOS,
-    prefersDarkModeObs(),
+    prefersColorSchemeDarkObs(),
-    (_use, prefs, syncWithOS, prefersDarkMode) => {
+    (_use, prefs, syncWithOS, prefersColorSchemeDark) => {
       if (syncWithOS) {
-        return prefersDarkMode ? 'dark' : 'light';
+        return prefersColorSchemeDark ? 'dark' : 'light';
       } else {
         return prefs.appearance;
       }

app/client/ui/createAppPage.ts

@@ -4,7 +4,8 @@ import {AppModel, TopAppModelImpl, TopAppModelOptions} from 'app/client/models/A
 import {reportError, setUpErrorHandling} from 'app/client/models/errors';
 import {buildSnackbarDom} from 'app/client/ui/NotifyUI';
 import {addViewportTag} from 'app/client/ui/viewport';
-import {attachCssRootVars, attachTheme} from 'app/client/ui2018/cssVars';
+import {attachCssRootVars} from 'app/client/ui2018/cssVars';
+import {attachTheme} from 'app/client/ui2018/theme';
 import {BaseAPI} from 'app/common/BaseAPI';
 import {dom, DomContents} from 'grainjs';
 
@@ -16,22 +17,22 @@ const G = getBrowserGlobals('document', 'window');
  */
 export function createAppPage(
   buildAppPage: (appModel: AppModel) => DomContents,
-  modelOptions: TopAppModelOptions = {}) {
+  modelOptions: TopAppModelOptions = {}
+) {
   setUpErrorHandling();
 
   const topAppModel = TopAppModelImpl.create(null, {}, undefined, modelOptions);
 
   addViewportTag();
   attachCssRootVars(topAppModel.productFlavor);
+  attachTheme();
   setupLocale().catch(reportError);
 
   // Add globals needed by test utils.
   G.window.gristApp = {
     testNumPendingApiRequests: () => BaseAPI.numPendingRequests(),
   };
-  dom.update(document.body, dom.maybeOwned(topAppModel.appObs, (owner, appModel) => {
+  dom.update(document.body, dom.maybe(topAppModel.appObs, (appModel) => {
-    owner.autoDispose(attachTheme(appModel.currentTheme));
-
     return [
       buildAppPage(appModel),
       buildSnackbarDom(appModel.notifier, appModel),

app/client/ui/createPage.ts

@@ -4,7 +4,8 @@ import {reportError, setErrorNotifier, setUpErrorHandling} from 'app/client/mode
 import {Notifier} from 'app/client/models/NotifyModel';
 import {buildSnackbarDom} from 'app/client/ui/NotifyUI';
 import {addViewportTag} from 'app/client/ui/viewport';
-import {attachCssRootVars, attachTheme, prefersColorSchemeThemeObs} from 'app/client/ui2018/cssVars';
+import {attachCssRootVars} from 'app/client/ui2018/cssVars';
+import {attachTheme} from 'app/client/ui2018/theme';
 import {BaseAPI} from 'app/common/BaseAPI';
 import {dom, DomContents} from 'grainjs';
 
@@ -21,6 +22,7 @@ export function createPage(buildPage: () => DomContents, options: {disableTheme?
 
   addViewportTag();
   attachCssRootVars('grist');
+  if (!disableTheme) { attachTheme(); }
   setupLocale().catch(reportError);
 
   // Add globals needed by test utils.
@@ -32,7 +34,6 @@ export function createPage(buildPage: () => DomContents, options: {disableTheme?
   setErrorNotifier(notifier);
 
   dom.update(document.body, () => [
-    disableTheme ? null : dom.autoDispose(attachTheme(prefersColorSchemeThemeObs())),
     buildPage(),
     buildSnackbarDom(notifier, null),
   ]);

app/client/ui/searchDropdown.ts

@@ -2,16 +2,17 @@
 // keyboard. Dropdown features a search input and reoders the list of
 // items to bring best matches at the top.
 
-import { Disposable, dom, DomElementMethod, IOptionFull, makeTestId, Observable, styled } from "grainjs";
-import { theme, vars } from 'app/client/ui2018/cssVars';
 import { ACIndexImpl, ACIndexOptions, ACItem, buildHighlightedDom, HighlightFunc,
          normalizeText } from "app/client/lib/ACIndex";
-import { menuDivider } from "app/client/ui2018/menus";
-import { icon } from "app/client/ui2018/icons";
-import { cssMenuItem, defaultMenuOptions, IOpenController, IPopupOptions, setPopupToFunc } from "popweasel";
-import { mergeWith } from "lodash";
-import { getOptionFull, SimpleList } from "../lib/simpleList";
 import { makeT } from 'app/client/lib/localization';
+import { getOptionFull, SimpleList } from "app/client/lib/simpleList";
+import { theme, vars } from 'app/client/ui2018/cssVars';
+import { icon } from "app/client/ui2018/icons";
+import { menuDivider } from "app/client/ui2018/menus";
+import { Disposable, dom, DomElementMethod, IOptionFull, makeTestId, Observable, styled } from "grainjs";
+import mergeWith from "lodash/mergeWith";
+import { cssMenuItem, defaultMenuOptions, IOpenController, IPopupOptions, setPopupToFunc } from "popweasel";
+
 
 const t = makeT('searchDropdown');
 

app/client/ui/tooltips.ts

@@ -582,7 +582,7 @@ const cssInfoTooltipPopup = styled('div', `
   display: flex;
   flex-direction: column;
   background-color: ${theme.popupBg};
-  max-width: 200px;
+  max-width: 240px;
   margin: 4px;
   padding: 0px;
 `);

app/client/ui2018/cssVars.ts

@@ -6,16 +6,10 @@
  *  https://css-tricks.com/snippets/css/system-font-stack/
  *
  */
-import {createPausableObs, PausableObservable} from 'app/client/lib/pausableObs';
-import {getStorage} from 'app/client/lib/storage';
 import {urlState} from 'app/client/models/gristUrlState';
 import {getTheme, ProductFlavor} from 'app/client/ui/CustomThemes';
-import {Theme, ThemeAppearance} from 'app/common/ThemePrefs';
+import {dom, DomElementMethod, makeTestId, Observable, styled, TestId} from 'grainjs';
-import {getThemeColors} from 'app/common/Themes';
-import {getGristConfig} from 'app/common/urlUtils';
-import {Computed, dom, DomElementMethod, makeTestId, Observable, styled, TestId} from 'grainjs';
 import debounce = require('lodash/debounce');
-import isEqual = require('lodash/isEqual');
 import values = require('lodash/values');
 
 const VAR_PREFIX = 'grist';
@@ -1021,51 +1015,6 @@ export function isScreenResizing(): Observable<boolean> {
   return _isScreenResizingObs;
 }
 
-export function prefersDarkMode(): boolean {
-  return window.matchMedia('(prefers-color-scheme: dark)').matches;
-}
-
-let _prefersDarkModeObs: PausableObservable<boolean>|undefined;
-
-/**
- * Returns a singleton observable for whether the user agent prefers dark mode.
- */
-export function prefersDarkModeObs(): PausableObservable<boolean> {
-  if (!_prefersDarkModeObs) {
-    const query = window.matchMedia('(prefers-color-scheme: dark)');
-    const obs = createPausableObs<boolean>(null, query.matches);
-    query.addEventListener('change', event => obs.set(event.matches));
-    _prefersDarkModeObs = obs;
-  }
-  return _prefersDarkModeObs;
-}
-
-let _prefersColorSchemeThemeObs: Computed<Theme>|undefined;
-
-/**
- * Returns a singleton observable for the Grist theme matching the current
- * user agent color scheme preference ("light" or "dark").
- */
-export function prefersColorSchemeThemeObs(): Computed<Theme> {
-  if (!_prefersColorSchemeThemeObs) {
-    const obs = Computed.create(null, prefersDarkModeObs(), (_use, prefersDarkTheme) => {
-      if (prefersDarkTheme) {
-        return {
-          appearance: 'dark',
-          colors: getThemeColors('GristDark'),
-        } as const;
-      } else {
-        return {
-          appearance: 'light',
-          colors: getThemeColors('GristLight'),
-        } as const;
-      }
-    });
-    _prefersColorSchemeThemeObs = obs;
-  }
-  return _prefersColorSchemeThemeObs;
-}
-
 /**
  * Attaches the global css properties to the document's root to make them available in the page.
  */
@@ -1081,96 +1030,6 @@ export function attachCssRootVars(productFlavor: ProductFlavor, varsOnly: boolea
   document.body.classList.add(`interface-${interfaceStyle}`);
 }
 
-export function attachTheme(themeObs: Observable<Theme>) {
-  // Attach the current theme to the DOM.
-  attachCssThemeVars(themeObs.get());
-
-  // Whenever the theme changes, re-attach it to the DOM.
-  return themeObs.addListener((newTheme, oldTheme) => {
-    if (isEqual(newTheme, oldTheme)) { return; }
-
-    attachCssThemeVars(newTheme);
-  });
-}
-
-/**
- * Attaches theme-related css properties to the theme style element.
- */
-function attachCssThemeVars({appearance, colors: themeColors}: Theme) {
-  // Custom CSS is incompatible with custom themes.
-  if (getGristConfig().enableCustomCss) { return; }
-
-  // Prepare the custom properties needed for applying the theme.
-  const properties = Object.entries(themeColors)
-    .map(([name, value]) => `--grist-theme-${name}: ${value};`);
-
-  // Include properties for styling the scrollbar.
-  properties.push(...getCssScrollbarProperties(appearance));
-
-  // Include properties for picking an appropriate background image.
-  properties.push(...getCssThemeBackgroundProperties(appearance));
-
-  // Apply the properties to the theme style element.
-  getOrCreateStyleElement('grist-theme').textContent = `:root {
-${properties.join('\n')}
-  }`;
-
-  // Make the browser aware of the color scheme.
-  document.documentElement.style.setProperty(`color-scheme`, appearance);
-
-  // Cache the appearance in local storage; this is currently used to apply a suitable
-  // background image that's shown while the application is loading.
-  getStorage().setItem('appearance', appearance);
-}
-
-/**
- * Gets scrollbar-related css properties that are appropriate for the given `appearance`.
- *
- * Note: Browser support for customizing scrollbars is still a mixed bag; the bulk of customization
- * is non-standard and unsupported by Firefox. If support matures, we could expose some of these in
- * custom themes, but for now we'll just go with reasonable presets.
- */
-function getCssScrollbarProperties(appearance: ThemeAppearance) {
-  return [
-    '--scroll-bar-fg: ' +
-      (appearance === 'dark' ? '#6B6B6B;' : '#A8A8A8;'),
-    '--scroll-bar-hover-fg: ' +
-      (appearance === 'dark' ? '#7B7B7B;' : '#8F8F8F;'),
-    '--scroll-bar-active-fg: ' +
-      (appearance === 'dark' ? '#8B8B8B;' : '#7C7C7C;'),
-    '--scroll-bar-bg: ' +
-      (appearance === 'dark' ? '#2B2B2B;' : '#F0F0F0;'),
-  ];
-}
-
-/**
- * Gets background-related css properties that are appropriate for the given `appearance`.
- *
- * Currently, this sets a property for showing a background image that's visible while a page
- * is loading.
- */
-function getCssThemeBackgroundProperties(appearance: ThemeAppearance) {
-  const value = appearance === 'dark'
-    ? 'url("img/prismpattern.png")'
-    : 'url("img/gplaypattern.png")';
-  return [`--grist-theme-bg: ${value};`];
-}
-
-/**
- * Gets or creates a style element in the head of the document with the given `id`.
- *
- * Useful for grouping CSS values such as theme custom properties without needing to
- * pollute the document with in-line styles.
- */
-function getOrCreateStyleElement(id: string) {
-  let style = document.head.querySelector(`#${id}`);
-  if (style) { return style; }
-  style = document.createElement('style');
-  style.setAttribute('id', id);
-  document.head.append(style);
-  return style;
-}
-
 // A dom method to hide element in print view
 export function hideInPrintView(): DomElementMethod {
   return cssHideInPrint.cls('');

app/client/ui2018/theme.ts

@@ -0,0 +1,191 @@
+import { createPausableObs, PausableObservable } from 'app/client/lib/pausableObs';
+import { getStorage } from 'app/client/lib/storage';
+import { urlState } from 'app/client/models/gristUrlState';
+import { Theme, ThemeAppearance, ThemeColors, ThemePrefs } from 'app/common/ThemePrefs';
+import { getThemeColors } from 'app/common/Themes';
+import { getGristConfig } from 'app/common/urlUtils';
+import { Computed, Observable } from 'grainjs';
+import isEqual from 'lodash/isEqual';
+
+const DEFAULT_LIGHT_THEME: Theme = {appearance: 'light', colors: getThemeColors('GristLight')};
+const DEFAULT_DARK_THEME: Theme = {appearance: 'dark', colors: getThemeColors('GristDark')};
+
+/**
+ * A singleton observable for the current user's Grist theme preferences.
+ *
+ * Set by `AppModel`, which populates it from `UserPrefs`.
+ */
+export const gristThemePrefs = Observable.create<ThemePrefs | null>(null, null);
+
+/**
+ * Returns `true` if the user agent prefers a dark color scheme.
+ */
+export function prefersColorSchemeDark(): boolean {
+  return window.matchMedia('(prefers-color-scheme: dark)').matches;
+}
+
+let _prefersColorSchemeDarkObs: PausableObservable<boolean> | undefined;
+
+/**
+ * Returns a singleton observable for whether the user agent prefers a
+ * dark color scheme.
+ */
+export function prefersColorSchemeDarkObs(): PausableObservable<boolean> {
+  if (!_prefersColorSchemeDarkObs) {
+    const query = window.matchMedia('(prefers-color-scheme: dark)');
+    const obs = createPausableObs<boolean>(null, query.matches);
+    query.addEventListener('change', event => obs.set(event.matches));
+    _prefersColorSchemeDarkObs = obs;
+  }
+  return _prefersColorSchemeDarkObs;
+}
+
+let _gristThemeObs: Computed<Theme> | undefined;
+
+/**
+ * A singleton observable for the current Grist theme.
+ */
+export function gristThemeObs() {
+  if (!_gristThemeObs) {
+    _gristThemeObs = Computed.create(null, (use) => {
+      // Custom CSS is incompatible with custom themes.
+      if (getGristConfig().enableCustomCss) { return DEFAULT_LIGHT_THEME; }
+
+      // If a user's preference is known, return it.
+      const themePrefs = use(gristThemePrefs);
+      const userAgentPrefersDarkTheme = use(prefersColorSchemeDarkObs());
+      if (themePrefs) { return getThemeFromPrefs(themePrefs, userAgentPrefersDarkTheme); }
+
+      // Otherwise, fall back to the user agent's preference.
+      return userAgentPrefersDarkTheme ? DEFAULT_DARK_THEME : DEFAULT_LIGHT_THEME;
+    });
+  }
+  return _gristThemeObs;
+}
+
+/**
+ * Attaches the current theme's CSS variables to the document, and
+ * re-attaches them whenever the theme changes.
+ */
+export function attachTheme() {
+  // Custom CSS is incompatible with custom themes.
+  if (getGristConfig().enableCustomCss) { return; }
+
+  // Attach the current theme's variables to the DOM.
+  attachCssThemeVars(gristThemeObs().get());
+
+  // Whenever the theme changes, re-attach its variables to the DOM.
+  gristThemeObs().addListener((newTheme, oldTheme) => {
+    if (isEqual(newTheme, oldTheme)) { return; }
+
+    attachCssThemeVars(newTheme);
+  });
+}
+
+/**
+ * Returns the `Theme` from the given `themePrefs`.
+ *
+ * If theme query parameters are present (`themeName`, `themeAppearance`, `themeSyncWithOs`),
+ * they will take precedence over their respective values in `themePrefs`.
+ */
+function getThemeFromPrefs(themePrefs: ThemePrefs, userAgentPrefersDarkTheme: boolean): Theme {
+  let {appearance, syncWithOS} = themePrefs;
+
+  const urlParams = urlState().state.get().params;
+  if (urlParams?.themeAppearance) {
+    appearance = urlParams?.themeAppearance;
+  }
+  if (urlParams?.themeSyncWithOs !== undefined) {
+    syncWithOS = urlParams?.themeSyncWithOs;
+  }
+
+  if (syncWithOS) {
+    appearance = userAgentPrefersDarkTheme ? 'dark' : 'light';
+  }
+
+  let nameOrColors = themePrefs.colors[appearance];
+  if (urlParams?.themeName) {
+    nameOrColors = urlParams?.themeName;
+  }
+
+  let colors: ThemeColors;
+  if (typeof nameOrColors === 'string') {
+    colors = getThemeColors(nameOrColors);
+  } else {
+    colors = nameOrColors;
+  }
+
+  return {appearance, colors};
+}
+
+function attachCssThemeVars({appearance, colors: themeColors}: Theme) {
+  // Prepare the custom properties needed for applying the theme.
+  const properties = Object.entries(themeColors)
+    .map(([name, value]) => `--grist-theme-${name}: ${value};`);
+
+  // Include properties for styling the scrollbar.
+  properties.push(...getCssThemeScrollbarProperties(appearance));
+
+  // Include properties for picking an appropriate background image.
+  properties.push(...getCssThemeBackgroundProperties(appearance));
+
+  // Apply the properties to the theme style element.
+  getOrCreateStyleElement('grist-theme').textContent = `:root {
+${properties.join('\n')}
+  }`;
+
+  // Make the browser aware of the color scheme.
+  document.documentElement.style.setProperty(`color-scheme`, appearance);
+
+  // Cache the appearance in local storage; this is currently used to apply a suitable
+  // background image that's shown while the application is loading.
+  getStorage().setItem('appearance', appearance);
+}
+
+/**
+ * Gets scrollbar-related css properties that are appropriate for the given `appearance`.
+ *
+ * Note: Browser support for customizing scrollbars is still a mixed bag; the bulk of customization
+ * is non-standard and unsupported by Firefox. If support matures, we could expose some of these in
+ * custom themes, but for now we'll just go with reasonable presets.
+ */
+function getCssThemeScrollbarProperties(appearance: ThemeAppearance) {
+  return [
+    '--scroll-bar-fg: ' +
+      (appearance === 'dark' ? '#6B6B6B;' : '#A8A8A8;'),
+    '--scroll-bar-hover-fg: ' +
+      (appearance === 'dark' ? '#7B7B7B;' : '#8F8F8F;'),
+    '--scroll-bar-active-fg: ' +
+      (appearance === 'dark' ? '#8B8B8B;' : '#7C7C7C;'),
+    '--scroll-bar-bg: ' +
+      (appearance === 'dark' ? '#2B2B2B;' : '#F0F0F0;'),
+  ];
+}
+
+/**
+ * Gets background-related css properties that are appropriate for the given `appearance`.
+ *
+ * Currently, this sets a property for showing a background image that's visible while a page
+ * is loading.
+ */
+function getCssThemeBackgroundProperties(appearance: ThemeAppearance) {
+  const value = appearance === 'dark'
+    ? 'url("img/prismpattern.png")'
+    : 'url("img/gplaypattern.png")';
+  return [`--grist-theme-bg: ${value};`];
+}
+
+/**
+ * Gets or creates a style element in the head of the document with the given `id`.
+ *
+ * Useful for grouping CSS values such as theme custom properties without needing to
+ * pollute the document with in-line styles.
+ */
+function getOrCreateStyleElement(id: string) {
+  let style = document.head.querySelector(`#${id}`);
+  if (style) { return style; }
+  style = document.createElement('style');
+  style.setAttribute('id', id);
+  document.head.append(style);
+  return style;
+}

app/client/widgets/ChoiceEditor.js

@@ -4,13 +4,22 @@ var TextEditor = require('app/client/widgets/TextEditor');
 
 const {Autocomplete} = require('app/client/lib/autocomplete');
 const {ACIndexImpl, buildHighlightedDom} = require('app/client/lib/ACIndex');
-const {ChoiceItem, cssChoiceList, cssMatchText, cssPlusButton,
+const {makeT} = require('app/client/lib/localization');
-       cssPlusIcon} = require('app/client/widgets/ChoiceListEditor');
+const {
+  buildDropdownConditionFilter,
+  ChoiceItem,
+  cssChoiceList,
+  cssMatchText,
+  cssPlusButton,
+  cssPlusIcon,
+} = require('app/client/widgets/ChoiceListEditor');
+const {icon} = require('app/client/ui2018/icons');
 const {menuCssClass} = require('app/client/ui2018/menus');
 const {testId, theme} = require('app/client/ui2018/cssVars');
 const {choiceToken, cssChoiceACItem} = require('app/client/widgets/ChoiceToken');
 const {dom, styled} = require('grainjs');
-const {icon} = require('../ui2018/icons');
+
+const t = makeT('ChoiceEditor');
 
 /**
  * ChoiceEditor - TextEditor with a dropdown for possible choices.
@@ -18,15 +27,46 @@ const {icon} = require('../ui2018/icons');
 function ChoiceEditor(options) {
   TextEditor.call(this, options);
 
-  this.choices = options.field.widgetOptionsJson.peek().choices || [];
+  this.widgetOptionsJson = options.field.widgetOptionsJson;
-  this.choiceOptions = options.field.widgetOptionsJson.peek().choiceOptions || {};
+  this.choices = this.widgetOptionsJson.peek().choices || [];
+  this.choicesSet = new Set(this.choices);
+  this.choiceOptions = this.widgetOptionsJson.peek().choiceOptions || {};
+
+  this.hasDropdownCondition = Boolean(options.field.dropdownCondition.peek()?.text);
+  this.dropdownConditionError;
+
+  let acItems = this.choices.map(c => new ChoiceItem(c, false, false));
+  if (this.hasDropdownCondition) {
+    try {
+      const dropdownConditionFilter = this.buildDropdownConditionFilter();
+      acItems = acItems.filter((item) => dropdownConditionFilter(item));
+    } catch (e) {
+      acItems = [];
+      this.dropdownConditionError = e.message;
+    }
+  }
+
+  const acIndex = new ACIndexImpl(acItems);
+  this._acOptions = {
+    popperOptions: {
+      placement: 'bottom'
+    },
+    menuCssClass: `${menuCssClass} ${cssChoiceList.className} test-autocomplete`,
+    buildNoItemsMessage: this.buildNoItemsMessage.bind(this),
+    search: (term) => this.maybeShowAddNew(acIndex.search(term), term),
+    renderItem: (item, highlightFunc) => this.renderACItem(item, highlightFunc),
+    getItemText: (item) => item.label,
+    onClick: () => this.options.commands.fieldEditSave(),
+  };
+
   if (!options.readonly && options.field.viewSection().parentKey() === "single") {
     this.cellEditorDiv.classList.add(cssChoiceEditor.className);
     this.cellEditorDiv.appendChild(cssChoiceEditIcon('Dropdown'));
   }
+
   // Whether to include a button to show a new choice.
   // TODO: Disable when the user cannot change column configuration.
-  this.enableAddNew = true;
+  this.enableAddNew = !this.hasDropdownCondition;
 }
 
 dispose.makeDisposable(ChoiceEditor);
@@ -66,20 +106,7 @@ ChoiceEditor.prototype.attach = function(cellElem) {
   // Don't create autocomplete if readonly.
   if (this.options.readonly) { return; }
 
-  const acItems = this.choices.map(c => new ChoiceItem(c, false, false));
+  this.autocomplete = Autocomplete.create(this, this.textInput, this._acOptions);
-  const acIndex = new ACIndexImpl(acItems);
-  const acOptions = {
-    popperOptions: {
-      placement: 'bottom'
-    },
-    menuCssClass: `${menuCssClass} ${cssChoiceList.className} test-autocomplete`,
-    search: (term) => this.maybeShowAddNew(acIndex.search(term), term),
-    renderItem: (item, highlightFunc) => this.renderACItem(item, highlightFunc),
-    getItemText: (item) => item.label,
-    onClick: () => this.options.commands.fieldEditSave(),
-  };
-
-  this.autocomplete = Autocomplete.create(this, this.textInput, acOptions);
 }
 
 /**
@@ -89,11 +116,35 @@ ChoiceEditor.prototype.attach = function(cellElem) {
 ChoiceEditor.prototype.prepForSave = async function() {
   const selectedItem = this.autocomplete && this.autocomplete.getSelectedItem();
   if (selectedItem && selectedItem.isNew) {
-    const choices = this.options.field.widgetOptionsJson.prop('choices');
+    const choices = this.widgetOptionsJson.prop('choices');
     await choices.saveOnly([...(choices.peek() || []), selectedItem.label]);
   }
 }
 
+ChoiceEditor.prototype.buildDropdownConditionFilter = function() {
+  const dropdownConditionCompiled = this.options.field.dropdownConditionCompiled.get();
+  if (dropdownConditionCompiled?.kind !== 'success') {
+    throw new Error('Dropdown condition is not compiled');
+  }
+
+  return buildDropdownConditionFilter({
+    dropdownConditionCompiled: dropdownConditionCompiled.result,
+    docData: this.options.gristDoc.docData,
+    tableId: this.options.field.tableId(),
+    rowId: this.options.rowId,
+  });
+}
+
+ChoiceEditor.prototype.buildNoItemsMessage = function() {
+  if (this.dropdownConditionError) {
+    return t('Error in dropdown condition');
+  } else if (this.hasDropdownCondition) {
+    return t('No choices matching condition');
+  } else {
+    return t('No choices to select');
+  }
+}
+
 /**
  * If the search text does not match anything exactly, adds 'new' item to it.
  *
@@ -103,15 +154,21 @@ ChoiceEditor.prototype.maybeShowAddNew = function(result, text) {
   // TODO: This logic is also mostly duplicated in ChoiceListEditor and ReferenceEditor.
   // See if there's anything common we can factor out and re-use.
   this.showAddNew = false;
+  if (!this.enableAddNew) {
+    return result;
+  }
+
   const trimmedText = text.trim();
-  if (!this.enableAddNew || !trimmedText) { return result; }
+  if (!trimmedText || this.choicesSet.has(trimmedText)) {
+    return result;
+  }
 
   const addNewItem = new ChoiceItem(trimmedText, false, false, true);
   if (result.items.find((item) => item.cleanText === addNewItem.cleanText)) {
     return result;
   }
 
-  result.items.push(addNewItem);
+  result.extraItems.push(addNewItem);
   this.showAddNew = true;
 
   return result;

app/client/widgets/ChoiceListEditor.ts

@@ -2,7 +2,9 @@ import {createGroup} from 'app/client/components/commands';
 import {ACIndexImpl, ACItem, ACResults,
         buildHighlightedDom, HighlightFunc, normalizeText} from 'app/client/lib/ACIndex';
 import {IAutocompleteOptions} from 'app/client/lib/autocomplete';
+import {makeT} from 'app/client/lib/localization';
 import {IToken, TokenField, tokenFieldStyles} from 'app/client/lib/TokenField';
+import {DocData} from 'app/client/models/DocData';
 import {colors, testId, theme} from 'app/client/ui2018/cssVars';
 import {menuCssClass} from 'app/client/ui2018/menus';
 import {createMobileButtons, getButtonMargins} from 'app/client/widgets/EditorButtons';
@@ -10,12 +12,15 @@ import {EditorPlacement} from 'app/client/widgets/EditorPlacement';
 import {FieldOptions, NewBaseEditor} from 'app/client/widgets/NewBaseEditor';
 import {csvEncodeRow} from 'app/common/csvFormat';
 import {CellValue} from "app/common/DocActions";
+import {CompiledPredicateFormula, EmptyRecordView} from 'app/common/PredicateFormula';
 import {decodeObject, encodeObject} from 'app/plugin/objtypes';
 import {ChoiceOptions, getRenderFillColor, getRenderTextColor} from 'app/client/widgets/ChoiceTextBox';
 import {choiceToken, cssChoiceACItem, cssChoiceToken} from 'app/client/widgets/ChoiceToken';
 import {icon} from 'app/client/ui2018/icons';
 import {dom, styled} from 'grainjs';
 
+const t = makeT('ChoiceListEditor');
+
 export class ChoiceItem implements ACItem, IToken {
   public cleanText: string = normalizeText(this.label);
   constructor(
@@ -38,25 +43,37 @@ export class ChoiceListEditor extends NewBaseEditor {
   private _inputSizer!: HTMLElement;     // Part of _contentSizer to size the text input
   private _alignment: string;
 
+  private _widgetOptionsJson = this.options.field.widgetOptionsJson.peek();
+  private _choices: string[] = this._widgetOptionsJson.choices || [];
+  private _choicesSet: Set<string> = new Set(this._choices);
+  private _choiceOptionsByName: ChoiceOptions = this._widgetOptionsJson.choiceOptions || {};
+
   // Whether to include a button to show a new choice.
   // TODO: Disable when the user cannot change column configuration.
-  private _enableAddNew: boolean = true;
+  private _enableAddNew: boolean;
   private _showAddNew: boolean = false;
 
-  private _choiceOptionsByName: ChoiceOptions;
+  private _hasDropdownCondition = Boolean(this.options.field.dropdownCondition.peek()?.text);
+  private _dropdownConditionError: string | undefined;
 
   constructor(protected options: FieldOptions) {
     super(options);
 
-    const choices: string[] = options.field.widgetOptionsJson.peek().choices || [];
+    let acItems = this._choices.map(c => new ChoiceItem(c, false, false));
-    this._choiceOptionsByName = options.field.widgetOptionsJson
+    if (this._hasDropdownCondition) {
-      .peek().choiceOptions || {};
+      try {
-    const acItems = choices.map(c => new ChoiceItem(c, false, false));
+        const dropdownConditionFilter = this._buildDropdownConditionFilter();
-    const choiceSet = new Set(choices);
+        acItems = acItems.filter((item) => dropdownConditionFilter(item));
+      } catch (e) {
+        acItems = [];
+        this._dropdownConditionError = e.message;
+      }
+    }
 
     const acIndex = new ACIndexImpl<ChoiceItem>(acItems);
     const acOptions: IAutocompleteOptions<ChoiceItem> = {
       menuCssClass: `${menuCssClass} ${cssChoiceList.className} test-autocomplete`,
+      buildNoItemsMessage: this._buildNoItemsMessage.bind(this),
       search: async (term: string) => this._maybeShowAddNew(acIndex.search(term), term),
       renderItem: (item, highlightFunc) => this._renderACItem(item, highlightFunc),
       getItemText: (item) => item.label,
@@ -65,12 +82,13 @@ export class ChoiceListEditor extends NewBaseEditor {
     this.commandGroup = this.autoDispose(createGroup(options.commands, null, true));
     this._alignment = options.field.widgetOptionsJson.peek().alignment || 'left';
 
+
     // If starting to edit by typing in a string, ignore previous tokens.
     const cellValue = decodeObject(options.cellValue);
     const startLabels: unknown[] = options.editValue !== undefined || !Array.isArray(cellValue) ? [] : cellValue;
     const startTokens = startLabels.map(label => new ChoiceItem(
       String(label),
-      !choiceSet.has(String(label)),
+      !this._choicesSet.has(String(label)),
       String(label).trim() === ''
     ));
 
@@ -87,7 +105,7 @@ export class ChoiceListEditor extends NewBaseEditor {
         cssChoiceToken.cls('-invalid', item.isInvalid),
         cssChoiceToken.cls('-blank', item.isBlank),
       ],
-      createToken: label => new ChoiceItem(label, !choiceSet.has(label), label.trim() === ''),
+      createToken: label => new ChoiceItem(label, !this._choicesSet.has(label), label.trim() === ''),
       acOptions,
       openAutocompleteOnFocus: true,
       readonly : options.readonly,
@@ -118,6 +136,8 @@ export class ChoiceListEditor extends NewBaseEditor {
       dom.prop('value', options.editValue || ''),
       this.commandGroup.attach(),
     );
+
+    this._enableAddNew = !this._hasDropdownCondition;
   }
 
   public attach(cellElem: Element): void {
@@ -150,7 +170,7 @@ export class ChoiceListEditor extends NewBaseEditor {
   }
 
   public getTextValue() {
-    const values = this._tokenField.tokensObs.get().map(t => t.label);
+    const values = this._tokenField.tokensObs.get().map(token => token.label);
     return csvEncodeRow(values, {prettier: true});
   }
 
@@ -164,7 +184,7 @@ export class ChoiceListEditor extends NewBaseEditor {
    */
   public async prepForSave() {
     const tokens = this._tokenField.tokensObs.get();
-    const newChoices = tokens.filter(t => t.isNew).map(t => t.label);
+    const newChoices = tokens.filter(({isNew}) => isNew).map(({label}) => label);
     if (newChoices.length > 0) {
       const choices = this.options.field.widgetOptionsJson.prop('choices');
       await choices.saveOnly([...(choices.peek() || []), ...new Set(newChoices)]);
@@ -218,6 +238,30 @@ export class ChoiceListEditor extends NewBaseEditor {
     this._textInput.style.width = this._inputSizer.getBoundingClientRect().width + 'px';
   }
 
+  private _buildDropdownConditionFilter() {
+    const dropdownConditionCompiled = this.options.field.dropdownConditionCompiled.get();
+    if (dropdownConditionCompiled?.kind !== 'success') {
+      throw new Error('Dropdown condition is not compiled');
+    }
+
+    return buildDropdownConditionFilter({
+      dropdownConditionCompiled: dropdownConditionCompiled.result,
+      docData: this.options.gristDoc.docData,
+      tableId: this.options.field.tableId(),
+      rowId: this.options.rowId,
+    });
+  }
+
+  private _buildNoItemsMessage(): string {
+    if (this._dropdownConditionError) {
+      return t('Error in dropdown condition');
+    } else if (this._hasDropdownCondition) {
+      return t('No choices matching condition');
+    } else {
+      return t('No choices to select');
+    }
+  }
+
   /**
    * If the search text does not match anything exactly, adds 'new' item to it.
    *
@@ -225,15 +269,21 @@ export class ChoiceListEditor extends NewBaseEditor {
    */
   private _maybeShowAddNew(result: ACResults<ChoiceItem>, text: string): ACResults<ChoiceItem> {
     this._showAddNew = false;
+    if (!this._enableAddNew) {
+      return result;
+    }
+
     const trimmedText = text.trim();
-    if (!this._enableAddNew || !trimmedText) { return result; }
+    if (!trimmedText || this._choicesSet.has(trimmedText)) {
+      return result;
+    }
 
     const addNewItem = new ChoiceItem(trimmedText, false, false, true);
     if (result.items.find((item) => item.cleanText === addNewItem.cleanText)) {
       return result;
     }
 
-    result.items.push(addNewItem);
+    result.extraItems.push(addNewItem);
     this._showAddNew = true;
 
     return result;
@@ -259,6 +309,24 @@ export class ChoiceListEditor extends NewBaseEditor {
   }
 }
 
+export interface GetACFilterFuncParams {
+  dropdownConditionCompiled: CompiledPredicateFormula;
+  docData: DocData;
+  tableId: string;
+  rowId: number;
+}
+
+export function buildDropdownConditionFilter(
+  params: GetACFilterFuncParams
+): (item: ChoiceItem) => boolean {
+  const {dropdownConditionCompiled, docData, tableId, rowId} = params;
+  const table = docData.getTable(tableId);
+  if (!table) { throw new Error(`Table ${tableId} not found`); }
+
+  const rec = table.getRecord(rowId) || new EmptyRecordView();
+  return (item: ChoiceItem) => dropdownConditionCompiled({rec, choice: item.label});
+}
+
 const cssCellEditor = styled('div', `
   background-color: ${theme.cellEditorBg};
   font-family: var(--grist-font-family-data);

app/client/widgets/ChoiceListEntry.ts

@@ -605,7 +605,6 @@ const cssButtonRow = styled('div', `
   gap: 8px;
   display: flex;
   margin-top: 8px;
-  margin-bottom: 16px;
 `);
 
 const cssDeleteButton = styled('div', `

app/client/widgets/ChoiceTextBox.ts

@@ -3,6 +3,7 @@ import {
   FormOptionsSortConfig,
   FormSelectConfig,
 } from 'app/client/components/Forms/FormConfig';
+import {DropdownConditionConfig} from 'app/client/components/DropdownConditionConfig';
 import {makeT} from 'app/client/lib/localization';
 import {DataRowModel} from 'app/client/models/DataRowModel';
 import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec';
@@ -82,11 +83,15 @@ export class ChoiceTextBox extends NTextBox {
     return [
       super.buildConfigDom(),
       this.buildChoicesConfigDom(),
+      dom.create(DropdownConditionConfig, this.field),
     ];
   }
 
   public buildTransformConfigDom() {
-    return this.buildConfigDom();
+    return [
+      super.buildConfigDom(),
+      this.buildChoicesConfigDom(),
+    ];
   }
 
   public buildFormConfigDom() {

app/client/widgets/ConditionalStyle.ts

@@ -4,7 +4,8 @@ import {ColumnRec} from 'app/client/models/DocModel';
 import {KoSaveableObservable} from 'app/client/models/modelUtil';
 import {RuleOwner} from 'app/client/models/RuleOwner';
 import {Style} from 'app/client/models/Styles';
-import {cssFieldFormula} from 'app/client/ui/FieldConfig';
+import {buildHighlightedCode} from 'app/client/ui/CodeHighlight';
+import {cssFieldFormula} from 'app/client/ui/RightPanelStyles';
 import {withInfoTooltip} from 'app/client/ui/tooltips';
 import {textButton} from 'app/client/ui2018/buttons';
 import {ColorOption, colorSelect} from 'app/client/ui2018/ColorSelect';
@@ -180,10 +181,11 @@ export class ConditionalStyle extends Disposable {
     column: ColumnRec,
     hasError: Observable<boolean>
   ) {
-    return cssFieldFormula(
+    return dom.create(buildHighlightedCode,
       formula,
-      { gristTheme: this._gristDoc.currentTheme, maxLines: 1 },
+      { maxLines: 1 },
       dom.cls('formula_field_sidepane'),
+      dom.cls(cssFieldFormula.className),
       dom.cls(cssErrorBorder.className, hasError),
       { tabIndex: '-1' },
       dom.on('focus', (_, refElem) => {

app/client/widgets/EditorButtons.ts

@@ -10,8 +10,8 @@ import {dom, styled} from 'grainjs';
 export function createMobileButtons(commands: IEditorCommandGroup) {
   // TODO A better check may be to detect a physical keyboard or touch support.
   return isDesktop() ? null : [
-    cssCancelBtn(cssIconWrap(cssFinishIcon('CrossSmall')), dom.on('click', commands.fieldEditCancel)),
+    cssCancelBtn(cssIconWrap(cssFinishIcon('CrossSmall')), dom.on('mousedown', commands.fieldEditCancel)),
-    cssSaveBtn(cssIconWrap(cssFinishIcon('Tick')), dom.on('click', commands.fieldEditSaveHere)),
+    cssSaveBtn(cssIconWrap(cssFinishIcon('Tick')), dom.on('mousedown', commands.fieldEditSaveHere)),
   ];
 }
 

app/client/widgets/FormulaAssistant.ts

@@ -9,12 +9,13 @@ import {ColumnRec, ViewFieldRec} from 'app/client/models/DocModel';
 import {ChatMessage} from 'app/client/models/entities/ColumnRec';
 import {HAS_FORMULA_ASSISTANT, WHICH_FORMULA_ASSISTANT} from 'app/client/models/features';
 import {getLoginOrSignupUrl, urlState} from 'app/client/models/gristUrlState';
-import {buildHighlightedCode} from 'app/client/ui/CodeHighlight';
+import {buildCodeHighlighter, buildHighlightedCode} from 'app/client/ui/CodeHighlight';
 import {autoGrow} from 'app/client/ui/forms';
 import {sanitizeHTML} from 'app/client/ui/sanitizeHTML';
 import {createUserImage} from 'app/client/ui/UserImage';
 import {basicButton, bigPrimaryButtonLink, primaryButton} from 'app/client/ui2018/buttons';
 import {theme, vars} from 'app/client/ui2018/cssVars';
+import {gristThemeObs} from 'app/client/ui2018/theme';
 import {icon} from 'app/client/ui2018/icons';
 import {cssLink} from 'app/client/ui2018/links';
 import {loadingDots} from 'app/client/ui2018/loaders';
@@ -1009,26 +1010,19 @@ class ChatHistory extends Disposable {
    * Renders the message as markdown if possible, otherwise as a code block.
    */
   private _render(message: string, ...args: DomElementArg[]) {
-    const doc = this._options.gristDoc;
     if (this.supportsMarkdown()) {
       return dom('div',
-        (el) => subscribeElem(el, doc.currentTheme, () => {
+        (el) => subscribeElem(el, gristThemeObs(), async () => {
+          const highlightCode = await buildCodeHighlighter({maxLines: 60});
           const content = sanitizeHTML(marked(message, {
-            highlight: (code) => {
+            highlight: (code) => highlightCode(code)
-              const codeBlock = buildHighlightedCode(code, {
-                gristTheme: doc.currentTheme,
-                maxLines: 60,
-              });
-              return codeBlock.innerHTML;
-            },
           }));
           el.innerHTML = content;
         }),
         ...args
       );
     } else {
-      return buildHighlightedCode(message, {
+      return dom.create(buildHighlightedCode, message, {
-        gristTheme: doc.currentTheme,
         maxLines: 100,
       });
     }

app/client/widgets/FormulaEditor.ts

@@ -74,15 +74,13 @@ export class FormulaEditor extends NewBaseEditor {
     this._aceEditor = AceEditor.create({
       // A bit awkward, but we need to assume calcSize is not used until attach() has been called
       // and _editorPlacement created.
-      column: options.column,
       calcSize: this._calcSize.bind(this),
-      gristDoc: options.gristDoc,
       saveValueOnBlurEvent: !options.readonly,
       editorState : this.editorState,
-      readonly: options.readonly
+      readonly: options.readonly,
+      getSuggestions: this._getSuggestions.bind(this),
     });
 
-
     // For editable editor we will grab the cursor when we are in the formula editing mode.
     const cursorCommands = options.readonly ? {} : { setCursor: this._onSetCursor };
     const isActive = Computed.create(this, use => Boolean(use(editingFormula)));
@@ -201,10 +199,7 @@ export class FormulaEditor extends NewBaseEditor {
       cssFormulaEditor.cls('-detached', this.isDetached),
       dom('div.formula_editor.formula_field_edit', testId('formula-editor'),
         this._aceEditor.buildDom((aceObj: any) => {
-          aceObj.setFontSize(11);
+          initializeAceOptions(aceObj);
-          aceObj.setHighlightActiveLine(false);
-          aceObj.getSession().setUseWrapMode(false);
-          aceObj.renderer.setPadding(0);
           const val = initialValue;
           const pos = Math.min(options.cursorPos, val.length);
           this._aceEditor.setValue(val, pos);
@@ -405,6 +400,17 @@ export class FormulaEditor extends NewBaseEditor {
     return result;
   }
 
+  private _getSuggestions(prefix: string) {
+    const section = this.options.gristDoc.viewModel.activeSection();
+    // If section is disposed or is pointing to an empty row, don't try to autocomplete.
+    if (!section?.getRowId()) { return []; }
+
+    const tableId = section.table().tableId();
+    const columnId = this.options.column.colId();
+    const rowId = section.activeRowId();
+    return this.options.gristDoc.docComm.autocomplete(prefix, tableId, columnId, rowId);
+  }
+
   // TODO: update regexes to unicode?
   private _onSetCursor(row?: DataRowModel, col?: ViewFieldRec) {
     // Don't do anything when we are readonly.
@@ -714,6 +720,13 @@ export function createFormulaErrorObs(owner: MultiHolder, gristDoc: GristDoc, or
   return errorMessage;
 }
 
+export function initializeAceOptions(aceObj: any) {
+  aceObj.setFontSize(11);
+  aceObj.setHighlightActiveLine(false);
+  aceObj.getSession().setUseWrapMode(false);
+  aceObj.renderer.setPadding(0);
+}
+
 const cssCollapseIcon = styled(icon, `
   margin: -3px 4px 0 4px;
   --icon-color: ${colors.slate};

app/client/widgets/Reference.ts

@@ -3,6 +3,7 @@ import {
   FormOptionsSortConfig,
   FormSelectConfig
 } from 'app/client/components/Forms/FormConfig';
+import {DropdownConditionConfig} from 'app/client/components/DropdownConditionConfig';
 import {makeT} from 'app/client/lib/localization';
 import {DataRowModel} from 'app/client/models/DataRowModel';
 import {TableRec} from 'app/client/models/DocModel';
@@ -55,6 +56,7 @@ export class Reference extends NTextBox {
   public buildConfigDom() {
     return [
       this.buildTransformConfigDom(),
+      dom.create(DropdownConditionConfig, this.field),
       cssLabel(t('CELL FORMAT')),
       super.buildConfigDom(),
     ];

app/client/widgets/ReferenceEditor.ts

@@ -11,7 +11,6 @@ import { nocaseEqual, ReferenceUtils } from 'app/client/lib/ReferenceUtils';
 import { undef } from 'app/common/gutil';
 import { styled } from 'grainjs';
 
-
 /**
  * A ReferenceEditor offers an autocomplete of choices from the referenced table.
  */
@@ -28,7 +27,12 @@ export class ReferenceEditor extends NTextEditor {
     this._utils = new ReferenceUtils(options.field, docData);
 
     const vcol = this._utils.visibleColModel;
-    this._enableAddNew = vcol && !vcol.isRealFormula() && !!vcol.colId();
+    this._enableAddNew = (
+      vcol &&
+      !vcol.isRealFormula() &&
+      !!vcol.colId() &&
+      !this._utils.hasDropdownCondition
+    );
 
     // Decorate the editor to look like a reference column value (with a "link" icon).
     // But not on readonly mode - here we will reuse default decoration
@@ -65,7 +69,8 @@ export class ReferenceEditor extends NTextEditor {
     // don't create autocomplete for readonly mode
     if (this.options.readonly) { return; }
     this._autocomplete = this.autoDispose(new Autocomplete<ICellItem>(this.textInput, {
-      menuCssClass: menuCssClass + ' ' + cssRefList.className,
+      menuCssClass: `${menuCssClass} ${cssRefList.className} test-autocomplete`,
+      buildNoItemsMessage: () => this._utils.buildNoItemsMessage(),
       search: this._doSearch.bind(this),
       renderItem: this._renderItem.bind(this),
       getItemText: (item) => item.text,
@@ -110,7 +115,7 @@ export class ReferenceEditor extends NTextEditor {
    * Also see: prepForSave.
    */
   private async _doSearch(text: string): Promise<ACResults<ICellItem>> {
-    const result = this._utils.autocompleteSearch(text);
+    const result = this._utils.autocompleteSearch(text, this.options.rowId);
 
     this._showAddNew = false;
     if (!this._enableAddNew || !text) { return result; }
@@ -120,7 +125,7 @@ export class ReferenceEditor extends NTextEditor {
       return result;
     }
 
-    result.items.push({rowId: 'new', text, cleanText});
+    result.extraItems.push({rowId: 'new', text, cleanText});
     this._showAddNew = true;
 
     return result;

app/client/widgets/ReferenceListEditor.ts

@@ -59,10 +59,16 @@ export class ReferenceListEditor extends NewBaseEditor {
     this._utils = new ReferenceUtils(options.field, docData);
 
     const vcol = this._utils.visibleColModel;
-    this._enableAddNew = vcol && !vcol.isRealFormula() && !!vcol.colId();
+    this._enableAddNew = (
+      vcol &&
+      !vcol.isRealFormula() &&
+      !!vcol.colId() &&
+      !this._utils.hasDropdownCondition
+    );
 
     const acOptions: IAutocompleteOptions<ReferenceItem> = {
-      menuCssClass: `${menuCssClass} ${cssRefList.className}`,
+      menuCssClass: `${menuCssClass} ${cssRefList.className} test-autocomplete`,
+      buildNoItemsMessage: () => this._utils.buildNoItemsMessage(),
       search: this._doSearch.bind(this),
       renderItem: this._renderItem.bind(this),
       getItemText: (item) => item.text,
@@ -166,12 +172,14 @@ export class ReferenceListEditor extends NewBaseEditor {
   }
 
   public getCellValue(): CellValue {
-    const rowIds = this._tokenField.tokensObs.get().map(t => typeof t.rowId === 'number' ? t.rowId : t.text);
+    const rowIds = this._tokenField.tokensObs.get()
+      .map(token => typeof token.rowId === 'number' ? token.rowId : token.text);
     return encodeObject(rowIds);
   }
 
   public getTextValue(): string {
-    const rowIds = this._tokenField.tokensObs.get().map(t => typeof t.rowId === 'number' ? String(t.rowId) : t.text);
+    const rowIds = this._tokenField.tokensObs.get()
+      .map(token => typeof token.rowId === 'number' ? String(token.rowId) : token.text);
     return csvEncodeRow(rowIds, {prettier: true});
   }
 
@@ -184,19 +192,19 @@ export class ReferenceListEditor extends NewBaseEditor {
    */
   public async prepForSave() {
     const tokens = this._tokenField.tokensObs.get();
-    const newValues = tokens.filter(t => t.rowId === 'new');
+    const newValues = tokens.filter(({rowId})=> rowId === 'new');
     if (newValues.length === 0) { return; }
 
     // Add the new items to the referenced table.
-    const colInfo = {[this._utils.visibleColId]: newValues.map(t => t.text)};
+    const colInfo = {[this._utils.visibleColId]: newValues.map(({text}) => text)};
     const rowIds = await this._utils.tableData.sendTableAction(
       ["BulkAddRecord", new Array(newValues.length).fill(null), colInfo]
     );
 
     // Update the TokenField tokens with the returned row ids.
     let i = 0;
-    const newTokens = tokens.map(t => {
+    const newTokens = tokens.map(token => {
-      return t.rowId === 'new' ? new ReferenceItem(t.text, rowIds[i++]) : t;
+      return token.rowId === 'new' ? new ReferenceItem(token.text, rowIds[i++]) : token;
     });
     this._tokenField.setTokens(newTokens);
   }
@@ -254,11 +262,12 @@ export class ReferenceListEditor extends NewBaseEditor {
    * Also see: prepForSave.
    */
    private async _doSearch(text: string): Promise<ACResults<ReferenceItem>> {
-    const {items, selectIndex, highlightFunc} = this._utils.autocompleteSearch(text);
+    const {items, selectIndex, highlightFunc} = this._utils.autocompleteSearch(text, this.options.rowId);
     const result: ACResults<ReferenceItem> = {
       selectIndex,
       highlightFunc,
-      items: items.map(i => new ReferenceItem(i.text, i.rowId))
+      items: items.map(i => new ReferenceItem(i.text, i.rowId)),
+      extraItems: [],
     };
 
     this._showAddNew = false;
@@ -269,7 +278,7 @@ export class ReferenceListEditor extends NewBaseEditor {
       return result;
     }
 
-    result.items.push(new ReferenceItem(text, 'new'));
+    result.extraItems.push(new ReferenceItem(text, 'new'));
     this._showAddNew = true;
 
     return result;

app/common/ACLRuleCollection.ts

@@ -3,14 +3,15 @@ import {AVAILABLE_BITS_COLUMNS, AVAILABLE_BITS_TABLES, trimPermissions} from 'ap
 import {ACLRulesReader} from 'app/common/ACLRulesReader';
 import {AclRuleProblem} from 'app/common/ActiveDocAPI';
 import {DocData} from 'app/common/DocData';
-import {AclMatchFunc, ParsedAclFormula, RulePart, RuleSet, UserAttributeRule} from 'app/common/GranularAccessClause';
+import {RulePart, RuleSet, UserAttributeRule} from 'app/common/GranularAccessClause';
 import {getSetMapValue, isNonNullish} from 'app/common/gutil';
+import {CompiledPredicateFormula, ParsedPredicateFormula} from 'app/common/PredicateFormula';
 import {MetaRowRecord} from 'app/common/TableData';
 import {decodeObject} from 'app/plugin/objtypes';
 
 export type ILogger = Pick<Console, 'log'|'debug'|'info'|'warn'|'error'>;
 
-const defaultMatchFunc: AclMatchFunc = () => true;
+const defaultMatchFunc: CompiledPredicateFormula = () => true;
 
 export const SPECIAL_RULES_TABLE_ID = '*SPECIAL';
 
@@ -20,12 +21,12 @@ const DEFAULT_RULE_SET: RuleSet = {
   colIds: '*',
   body: [{
     aclFormula: "user.Access in [EDITOR, OWNER]",
-    matchFunc:  (input) => ['editors', 'owners'].includes(String(input.user.Access)),
+    matchFunc: (input) => ['editors', 'owners'].includes(String(input.user!.Access)),
     permissions: parsePermissions('all'),
     permissionsText: 'all',
   }, {
     aclFormula: "user.Access in [VIEWER]",
-    matchFunc:  (input) => ['viewers'].includes(String(input.user.Access)),
+    matchFunc: (input) => ['viewers'].includes(String(input.user!.Access)),
     permissions: parsePermissions('+R-CUDS'),
     permissionsText: '+R',
   }, {
@@ -48,7 +49,7 @@ const SPECIAL_RULE_SETS: Record<string, RuleSet> = {
     colIds: ['SchemaEdit'],
     body: [{
       aclFormula: "user.Access in [EDITOR, OWNER]",
-      matchFunc:  (input) => ['editors', 'owners'].includes(String(input.user.Access)),
+      matchFunc: (input) => ['editors', 'owners'].includes(String(input.user!.Access)),
       permissions: parsePermissions('+S'),
       permissionsText: '+S',
     }, {
@@ -63,7 +64,7 @@ const SPECIAL_RULE_SETS: Record<string, RuleSet> = {
     colIds: ['AccessRules'],
     body: [{
       aclFormula: "user.Access in [OWNER]",
-      matchFunc:  (input) => ['owners'].includes(String(input.user.Access)),
+      matchFunc: (input) => ['owners'].includes(String(input.user!.Access)),
       permissions: parsePermissions('+R'),
       permissionsText: '+R',
     }, {
@@ -78,7 +79,7 @@ const SPECIAL_RULE_SETS: Record<string, RuleSet> = {
     colIds: ['FullCopies'],
     body: [{
       aclFormula: "user.Access in [OWNER]",
-      matchFunc:  (input) => ['owners'].includes(String(input.user.Access)),
+      matchFunc: (input) => ['owners'].includes(String(input.user!.Access)),
       permissions: parsePermissions('+R'),
       permissionsText: '+R',
     }, {
@@ -102,7 +103,7 @@ const EMERGENCY_RULE_SET: RuleSet = {
   colIds: '*',
   body: [{
     aclFormula: "user.Access in [OWNER]",
-    matchFunc:  (input) => ['owners'].includes(String(input.user.Access)),
+    matchFunc:  (input) => ['owners'].includes(String(input.user!.Access)),
     permissions: parsePermissions('all'),
     permissionsText: 'all',
   }, {
@@ -381,7 +382,7 @@ export class ACLRuleCollection {
 
 export interface ReadAclOptions {
   log: ILogger;     // For logging warnings during rule processing.
-  compile?: (parsed: ParsedAclFormula) => AclMatchFunc;
+  compile?: (parsed: ParsedPredicateFormula) => CompiledPredicateFormula;
   // If true, add and modify access rules in some special ways.
   // Specifically, call addHelperCols to add helper columns of restricted columns to rule sets,
   // and use ACLShareRules to implement any special shares as access rules.

app/common/ACLRulesReader.ts

@@ -3,7 +3,8 @@ import { getSetMapValue } from 'app/common/gutil';
 import { SchemaTypes } from 'app/common/schema';
 import { ShareOptions } from 'app/common/ShareOptions';
 import { MetaRowRecord, MetaTableData } from 'app/common/TableData';
-import { isEqual, sortBy } from 'lodash';
+import isEqual from 'lodash/isEqual';
+import sortBy from 'lodash/sortBy';
 
 /**
  * For special shares, we need to refer to resources that may not

app/common/ActiveDocAPI.ts

@@ -1,6 +1,6 @@
 import {ActionGroup} from 'app/common/ActionGroup';
 import {BulkAddRecord, CellValue, TableDataAction, UserAction} from 'app/common/DocActions';
-import {FormulaProperties} from 'app/common/GranularAccessClause';
+import {PredicateFormulaProperties} from 'app/common/PredicateFormula';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {DocStateComparison, PermissionData, UserAccessData} from 'app/common/UserAPI';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
@@ -421,7 +421,7 @@ export interface ActiveDocAPI {
    * Find and return a list of auto-complete suggestions that start with `txt`, when editing a
    * formula in table `tableId` and column `columnId`.
    */
-  autocomplete(txt: string, tableId: string, columnId: string, rowId: UIRowId): Promise<ISuggestionWithValue[]>;
+  autocomplete(txt: string, tableId: string, columnId: string, rowId: UIRowId | null): Promise<ISuggestionWithValue[]>;
 
   /**
    * Removes the current instance from the doc.
@@ -467,7 +467,7 @@ export interface ActiveDocAPI {
   /**
    * Check if an ACL formula is valid. If not, will throw an error with an explanation.
    */
-  checkAclFormula(text: string): Promise<FormulaProperties>;
+  checkAclFormula(text: string): Promise<PredicateFormulaProperties>;
 
   /**
    * Get a token for out-of-band access to the document.

app/common/ColumnFilterFunc.ts

@@ -4,7 +4,7 @@ import {decodeObject} from "app/plugin/objtypes";
 import moment, { Moment } from "moment-timezone";
 import {extractInfoFromColType, isDateLikeType, isList, isListType, isNumberType} from "app/common/gristTypes";
 import {isRelativeBound, relativeDateToUnixTimestamp} from "app/common/RelativeDates";
-import {noop} from "lodash";
+import noop from "lodash/noop";
 
 export type ColumnFilterFunc = (value: CellValue) => boolean;
 

app/common/DropdownCondition.ts

@@ -0,0 +1,20 @@
+import { CompiledPredicateFormula } from 'app/common/PredicateFormula';
+
+export interface DropdownCondition {
+  text: string;
+  parsed: string;
+}
+
+export type DropdownConditionCompilationResult =
+  | DropdownConditionCompilationSuccess
+  | DropdownConditionCompilationFailure;
+
+interface DropdownConditionCompilationSuccess {
+  kind: 'success';
+  result: CompiledPredicateFormula;
+}
+
+interface DropdownConditionCompilationFailure {
+  kind: 'failure';
+  error: string;
+}

app/common/GranularAccessClause.ts

@@ -1,7 +1,8 @@
 import {PartialPermissionSet} from 'app/common/ACLPermissions';
 import {CellValue, RowRecord} from 'app/common/DocActions';
+import {CompiledPredicateFormula} from 'app/common/PredicateFormula';
+import {Role} from 'app/common/roles';
 import {MetaRowRecord} from 'app/common/TableData';
-import {Role} from './roles';
 
 export interface RuleSet {
   tableId: '*' | string;
@@ -18,7 +19,7 @@ export interface RulePart {
   permissionsText: string;        // The text version of PermissionSet, as stored.
 
   // Compiled version of aclFormula.
-  matchFunc?: AclMatchFunc;
+  matchFunc?: CompiledPredicateFormula;
 
   // Optional memo, currently extracted from comment in formula.
   memo?: string;
@@ -53,35 +54,6 @@ export interface UserInfo {
   toJSON(): {[key: string]: any};
 }
 
-/**
- * Input into the AclMatchFunc. Compiled formulas evaluate AclMatchInput to produce a boolean.
- */
-export interface AclMatchInput {
-  user: UserInfo;
-  rec?: InfoView;
-  newRec?: InfoView;
-  docId?: string;
-}
-
-/**
- * The actual boolean function that can evaluate a request. The result of compiling ParsedAclFormula.
- */
-export type AclMatchFunc = (input: AclMatchInput) => boolean;
-
-/**
- * Representation of a parsed ACL formula.
- */
-type PrimitiveCellValue = number|string|boolean|null;
-export type ParsedAclFormula = [string, ...(ParsedAclFormula|PrimitiveCellValue)[]];
-
-/**
- * Observations about a formula.
- */
-export interface FormulaProperties {
-  hasRecOrNewRec?: boolean;
-  usedColIds?: string[];
-}
-
 export interface UserAttributeRule {
   origRecord?: RowRecord;         // Original record used to create this UserAttributeRule.
   name: string;       // Should be unique among UserAttributeRules.
@@ -89,45 +61,3 @@ export interface UserAttributeRule {
   lookupColId: string;  // Column in tableId in which to do the lookup.
   charId: string;     // Attribute to look up, possibly a path. E.g. 'Email' or 'office.city'.
 }
-
-/**
- * Check some key facts about the formula.
- */
-export function getFormulaProperties(formula: ParsedAclFormula) {
-  const result: FormulaProperties = {};
-  if (usesRec(formula)) { result.hasRecOrNewRec = true; }
-  const colIds = new Set<string>();
-  collectRecColIds(formula, colIds);
-  result.usedColIds = Array.from(colIds);
-  return result;
-}
-
-/**
- * Check whether a formula mentions `rec` or `newRec`.
- */
-export function usesRec(formula: ParsedAclFormula): boolean {
-  if (!Array.isArray(formula)) { throw new Error('expected a list'); }
-  if (isRecOrNewRec(formula)) {
-    return true;
-  }
-  return formula.some(el => {
-    if (!Array.isArray(el)) { return false; }
-    return usesRec(el);
-  });
-}
-
-function isRecOrNewRec(formula: ParsedAclFormula|PrimitiveCellValue): boolean {
-  return Array.isArray(formula) &&
-    formula[0] === 'Name' &&
-    (formula[1] === 'rec' || formula[1] === 'newRec');
-}
-
-function collectRecColIds(formula: ParsedAclFormula, colIds: Set<string>): void {
-  if (!Array.isArray(formula)) { throw new Error('expected a list'); }
-  if (formula[0] === 'Attr' && isRecOrNewRec(formula[1])) {
-    const colId = formula[2];
-    colIds.add(String(colId));
-    return;
-  }
-  formula.forEach(el => Array.isArray(el) && collectRecColIds(el, colIds));
-}

app/common/PredicateFormula.ts

@@ -0,0 +1,223 @@
+/**
+ * Representation and compilation of predicate formulas.
+ *
+ * An example of a predicate formula is: "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']".
+ * These formulas are parsed in Python into a tree with nodes of the form [NODE_TYPE, ...args].
+ * See sandbox/grist/predicate_formula.py for details.
+ *
+ * This module includes typings for the nodes, and the compilePredicateFormula() function that
+ * turns such trees into actual predicate functions.
+ */
+import {CellValue, RowRecord} from 'app/common/DocActions';
+import {ErrorWithCode} from 'app/common/ErrorWithCode';
+import {InfoView, UserInfo} from 'app/common/GranularAccessClause';
+import {decodeObject} from 'app/plugin/objtypes';
+import constant = require('lodash/constant');
+
+/**
+ * Representation of a parsed predicate formula.
+ */
+export type PrimitiveCellValue = number|string|boolean|null;
+export type ParsedPredicateFormula = [string, ...(ParsedPredicateFormula|PrimitiveCellValue)[]];
+
+/**
+ * Inputs to a predicate formula function.
+ */
+export interface PredicateFormulaInput {
+  user?: UserInfo;
+  rec?: RowRecord|InfoView;
+  newRec?: InfoView;
+  docId?: string;
+  choice?: string|RowRecord|InfoView;
+}
+
+export class EmptyRecordView implements InfoView {
+  public get(_colId: string): CellValue { return null; }
+  public toJSON() { return {}; }
+}
+
+/**
+ * The result of compiling ParsedPredicateFormula.
+ */
+export type CompiledPredicateFormula = (input: PredicateFormulaInput) => boolean;
+
+const GRIST_CONSTANTS: Record<string, string> = {
+  EDITOR: 'editors',
+  OWNER: 'owners',
+  VIEWER: 'viewers',
+};
+
+/**
+ * An intermediate predicate formula returned during compilation, which may return
+ * a non-boolean value.
+ */
+type IntermediatePredicateFormula = (input: PredicateFormulaInput) => any;
+
+export interface CompilePredicateFormulaOptions {
+  /** Defaults to `'acl'`. */
+  variant?: 'acl'|'dropdown-condition';
+}
+
+/**
+ * Compiles a parsed predicate formula and returns it.
+ */
+export function compilePredicateFormula(
+  parsedPredicateFormula: ParsedPredicateFormula,
+  options: CompilePredicateFormulaOptions = {}
+): CompiledPredicateFormula {
+  const {variant = 'acl'} = options;
+
+  function compileNode(node: ParsedPredicateFormula): IntermediatePredicateFormula {
+    const rawArgs = node.slice(1);
+    const args = rawArgs as ParsedPredicateFormula[];
+    switch (node[0]) {
+      case 'And':   { const parts = args.map(compileNode); return (input) => parts.every(p => p(input)); }
+      case 'Or':    { const parts = args.map(compileNode); return (input) => parts.some(p => p(input)); }
+      case 'Add':   return compileAndCombine(args, ([a, b]) => a + b);
+      case 'Sub':   return compileAndCombine(args, ([a, b]) => a - b);
+      case 'Mult':  return compileAndCombine(args, ([a, b]) => a * b);
+      case 'Div':   return compileAndCombine(args, ([a, b]) => a / b);
+      case 'Mod':   return compileAndCombine(args, ([a, b]) => a % b);
+      case 'Not':   return compileAndCombine(args, ([a]) => !a);
+      case 'Eq':    return compileAndCombine(args, ([a, b]) => a === b);
+      case 'NotEq': return compileAndCombine(args, ([a, b]) => a !== b);
+      case 'Lt':    return compileAndCombine(args, ([a, b]) => a < b);
+      case 'LtE':   return compileAndCombine(args, ([a, b]) => a <= b);
+      case 'Gt':    return compileAndCombine(args, ([a, b]) => a > b);
+      case 'GtE':   return compileAndCombine(args, ([a, b]) => a >= b);
+      case 'Is':    return compileAndCombine(args, ([a, b]) => a === b);
+      case 'IsNot': return compileAndCombine(args, ([a, b]) => a !== b);
+      case 'In':    return compileAndCombine(args, ([a, b]) => Boolean(b?.includes(a)));
+      case 'NotIn': return compileAndCombine(args, ([a, b]) => !b?.includes(a));
+      case 'List':  return compileAndCombine(args, (values) => values);
+      case 'Const': return constant(node[1] as CellValue);
+      case 'Name': {
+        const name = rawArgs[0] as keyof PredicateFormulaInput;
+        if (GRIST_CONSTANTS[name]) { return constant(GRIST_CONSTANTS[name]); }
+
+        let validNames: string[];
+        switch (variant) {
+          case 'acl': {
+            validNames = ['newRec', 'rec', 'user'];
+            break;
+          }
+          case 'dropdown-condition': {
+            validNames = ['rec', 'choice'];
+            break;
+          }
+        }
+        if (!validNames.includes(name)) { throw new Error(`Unknown variable '${name}'`); }
+
+        return (input) => input[name];
+      }
+      case 'Attr': {
+        const attrName = rawArgs[1] as string;
+        return compileAndCombine([args[0]], ([value]) => getAttr(value, attrName, args[0]));
+      }
+      case 'Comment': return compileNode(args[0]);
+    }
+    throw new Error(`Unknown node type '${node[0]}'`);
+  }
+
+  /**
+   * Helper for operators: compile a list of nodes, then when evaluating, evaluate them all and
+   * combine the array of results using the given combine() function.
+   */
+  function compileAndCombine(
+    args: ParsedPredicateFormula[],
+    combine: (values: any[]) => any
+  ): IntermediatePredicateFormula {
+    const compiled = args.map(compileNode);
+    return (input: PredicateFormulaInput) => combine(compiled.map(c => c(input)));
+  }
+
+  const compiledPredicateFormula = compileNode(parsedPredicateFormula);
+  return (input) => Boolean(compiledPredicateFormula(input));
+}
+
+function describeNode(node: ParsedPredicateFormula): string {
+  if (node[0] === 'Name') {
+    return node[1] as string;
+  } else if (node[0] === 'Attr') {
+    return describeNode(node[1] as ParsedPredicateFormula) + '.' + (node[2] as string);
+  } else {
+    return 'value';
+  }
+}
+
+function getAttr(value: any, attrName: string, valueNode: ParsedPredicateFormula): any {
+  if (value == null) {
+    if (valueNode[0] === 'Name' && (valueNode[1] === 'rec' || valueNode[1] === 'newRec')) {
+      // This code is recognized by GranularAccess to know when an ACL rule is row-specific.
+      throw new ErrorWithCode('NEED_ROW_DATA', `Missing row data '${valueNode[1]}'`);
+    }
+    throw new Error(`No value for '${describeNode(valueNode)}'`);
+  }
+  return typeof value.get === 'function'
+    ? decodeObject(value.get(attrName)) // InfoView
+    : value[attrName];
+}
+
+/**
+ * Predicate formula properties.
+ */
+export interface PredicateFormulaProperties {
+  /**
+   * List of column ids that are referenced by either `$` or `rec.` notation.
+   */
+  recColIds?: string[];
+  /**
+   * List of column ids that are referenced by `choice.` notation.
+   *
+   * Only applies to the `dropdown-condition` variant of predicate formulas,
+   * and only for Reference and Reference List columns.
+   */
+  choiceColIds?: string[];
+}
+
+/**
+ * Returns properties about a predicate `formula`.
+ *
+ * Properties include the list of column ids referenced in the formula.
+ * Currently, this information is used for error validation; specifically, to
+ * report when invalid column ids are referenced in ACL formulas and dropdown
+ * conditions.
+ */
+export function getPredicateFormulaProperties(
+  formula: ParsedPredicateFormula
+): PredicateFormulaProperties {
+  return {
+    recColIds: [...getRecColIds(formula)],
+    choiceColIds: [...getChoiceColIds(formula)],
+  };
+}
+
+function isRecOrNewRec(formula: ParsedPredicateFormula|PrimitiveCellValue): boolean {
+  return Array.isArray(formula) &&
+    formula[0] === 'Name' &&
+    (formula[1] === 'rec' || formula[1] === 'newRec');
+}
+
+function getRecColIds(formula: ParsedPredicateFormula): string[] {
+  return [...new Set(collectColIds(formula, isRecOrNewRec))];
+}
+
+function isChoice(formula: ParsedPredicateFormula|PrimitiveCellValue): boolean {
+  return Array.isArray(formula) && formula[0] === 'Name' && formula[1] === 'choice';
+}
+
+function getChoiceColIds(formula: ParsedPredicateFormula): string[] {
+  return [...new Set(collectColIds(formula, isChoice))];
+}
+
+function collectColIds(
+  formula: ParsedPredicateFormula,
+  isIdentifierWithColIds: (formula: ParsedPredicateFormula|PrimitiveCellValue) => boolean,
+): string[] {
+  if (!Array.isArray(formula)) { throw new Error('expected a list'); }
+  if (formula[0] === 'Attr' && isIdentifierWithColIds(formula[1])) {
+    const colId = String(formula[2]);
+    return [colId];
+  }
+  return formula.flatMap(el => Array.isArray(el) ? collectColIds(el, isIdentifierWithColIds) : []);
+}

app/common/RelativeDates.ts

@@ -2,9 +2,12 @@
 // time defined as a series of periods. Hence, starting from the current date, each one of the
 // periods gets applied successively which eventually yields to the final date. Typical relative
 
-import { isEqual, isNumber, isUndefined, omitBy } from "lodash";
-import moment from "moment-timezone";
 import getCurrentTime from "app/common/getCurrentTime";
+import isEqual from "lodash/isEqual";
+import isNumber from "lodash/isNumber";
+import isUndefined from "lodash/isUndefined";
+import omitBy from "lodash/omitBy";
+import moment from "moment-timezone";
 
 // Relative date uses one or two periods. When relative dates are defined by two periods, they are
 // applied successively to the start date to resolve the target date. In practice in grist, as of

app/server/lib/ACLFormula.ts

@@ -1,109 +0,0 @@
-/**
- * Representation and compilation of ACL formulas.
- *
- * An example of an ACL formula is: "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']".
- * These formulas are parsed in Python into a tree with nodes of the form [NODE_TYPE, ...args].
- * See sandbox/grist/acl_formula.py for details.
- *
- * This modules includes typings for the nodes, and compileAclFormula() function that turns such a
- * tree into an actual boolean function.
- */
-import {CellValue} from 'app/common/DocActions';
-import {ErrorWithCode} from 'app/common/ErrorWithCode';
-import {AclMatchFunc, AclMatchInput, ParsedAclFormula} from 'app/common/GranularAccessClause';
-import {decodeObject} from "app/plugin/objtypes";
-import constant = require('lodash/constant');
-
-const GRIST_CONSTANTS: Record<string, string> = {
-  EDITOR: 'editors',
-  OWNER: 'owners',
-  VIEWER: 'viewers',
-};
-
-/**
- * Compile a parsed ACL formula into an actual function that can evaluate a request.
- */
-export function compileAclFormula(parsedAclFormula: ParsedAclFormula): AclMatchFunc {
-  const compiled = _compileNode(parsedAclFormula);
-  return (input) => Boolean(compiled(input));
-}
-
-/**
- * Type for intermediate functions, which may return values other than booleans.
- */
-type AclEvalFunc = (input: AclMatchInput) => any;
-
-/**
- * Compile a single node of the parsed formula tree.
- */
-function _compileNode(parsedAclFormula: ParsedAclFormula): AclEvalFunc {
-  const rawArgs = parsedAclFormula.slice(1);
-  const args = rawArgs as ParsedAclFormula[];
-  switch (parsedAclFormula[0]) {
-    case 'And':   { const parts = args.map(_compileNode); return (input) => parts.every(p => p(input)); }
-    case 'Or':    { const parts = args.map(_compileNode); return (input) => parts.some(p => p(input)); }
-    case 'Add':   return _compileAndCombine(args, ([a, b]) => a + b);
-    case 'Sub':   return _compileAndCombine(args, ([a, b]) => a - b);
-    case 'Mult':  return _compileAndCombine(args, ([a, b]) => a * b);
-    case 'Div':   return _compileAndCombine(args, ([a, b]) => a / b);
-    case 'Mod':   return _compileAndCombine(args, ([a, b]) => a % b);
-    case 'Not':   return _compileAndCombine(args, ([a]) => !a);
-    case 'Eq':    return _compileAndCombine(args, ([a, b]) => a === b);
-    case 'NotEq': return _compileAndCombine(args, ([a, b]) => a !== b);
-    case 'Lt':    return _compileAndCombine(args, ([a, b]) => a < b);
-    case 'LtE':   return _compileAndCombine(args, ([a, b]) => a <= b);
-    case 'Gt':    return _compileAndCombine(args, ([a, b]) => a > b);
-    case 'GtE':   return _compileAndCombine(args, ([a, b]) => a >= b);
-    case 'Is':    return _compileAndCombine(args, ([a, b]) => a === b);
-    case 'IsNot': return _compileAndCombine(args, ([a, b]) => a !== b);
-    case 'In':    return _compileAndCombine(args, ([a, b]) => Boolean(b?.includes(a)));
-    case 'NotIn': return _compileAndCombine(args, ([a, b]) => !b?.includes(a));
-    case 'List':  return _compileAndCombine(args, (values) => values);
-    case 'Const': return constant(parsedAclFormula[1] as CellValue);
-    case 'Name': {
-      const name = rawArgs[0] as keyof AclMatchInput;
-      if (GRIST_CONSTANTS[name]) { return constant(GRIST_CONSTANTS[name]); }
-      if (!['user', 'rec', 'newRec'].includes(name)) {
-        throw new Error(`Unknown variable '${name}'`);
-      }
-      return (input) => input[name];
-    }
-    case 'Attr': {
-      const attrName = rawArgs[1] as string;
-      return _compileAndCombine([args[0]], ([value]) => getAttr(value, attrName, args[0]));
-    }
-    case 'Comment': return _compileNode(args[0]);
-  }
-  throw new Error(`Unknown node type '${parsedAclFormula[0]}'`);
-}
-
-function describeNode(node: ParsedAclFormula): string {
-  if (node[0] === 'Name') {
-    return node[1] as string;
-  } else if (node[0] === 'Attr') {
-    return describeNode(node[1] as ParsedAclFormula) + '.' + (node[2] as string);
-  } else {
-    return 'value';
-  }
-}
-
-function getAttr(value: any, attrName: string, valueNode: ParsedAclFormula): any {
-  if (value == null) {
-    if (valueNode[0] === 'Name' && (valueNode[1] === 'rec' || valueNode[1] === 'newRec')) {
-      // This code is recognized by GranularAccess to know when a rule is row-specific.
-      throw new ErrorWithCode('NEED_ROW_DATA', `Missing row data '${valueNode[1]}'`);
-    }
-    throw new Error(`No value for '${describeNode(valueNode)}'`);
-  }
-  return (typeof value.get === 'function' ? decodeObject(value.get(attrName)) : // InfoView
-          value[attrName]);
-}
-
-/**
- * Helper for operators: compile a list of nodes, then when evaluating, evaluate them all and
- * combine the array of results using the given combine() function.
- */
-function _compileAndCombine(args: ParsedAclFormula[], combine: (values: any[]) => any): AclEvalFunc {
-  const compiled = args.map(_compileNode);
-  return (input: AclMatchInput) => combine(compiled.map(c => c(input)));
-}

app/server/lib/ActiveDoc.ts

@@ -64,12 +64,16 @@ import {
 } from 'app/common/DocUsage';
 import {normalizeEmail} from 'app/common/emails';
 import {Product} from 'app/common/Features';
-import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
 import {isHiddenCol} from 'app/common/gristTypes';
 import {commonUrls, parseUrlId} from 'app/common/gristUrls';
 import {byteString, countIf, retryOnce, safeJsonParse, timeoutReached} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import {Interval} from 'app/common/Interval';
+import {
+  compilePredicateFormula,
+  getPredicateFormulaProperties,
+  PredicateFormulaProperties,
+} from 'app/common/PredicateFormula';
 import * as roles from 'app/common/roles';
 import {schema, SCHEMA_VERSION} from 'app/common/schema';
 import {MetaRowRecord, SingleCell} from 'app/common/TableData';
@@ -84,7 +88,6 @@ import {Share} from 'app/gen-server/entity/Share';
 import {RecordWithStringId} from 'app/plugin/DocApiTypes';
 import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI';
 import {AccessTokenOptions, AccessTokenResult, GristDocAPI, UIRowId} from 'app/plugin/GristAPI';
-import {compileAclFormula} from 'app/server/lib/ACLFormula';
 import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance';
 import {AssistanceContext} from 'app/common/AssistancePrompts';
 import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer';
@@ -1289,7 +1292,11 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public async autocomplete(
-    docSession: DocSession, txt: string, tableId: string, columnId: string, rowId: UIRowId
+    docSession: DocSession,
+    txt: string,
+    tableId: string,
+    columnId: string,
+    rowId: UIRowId | null
   ): Promise<ISuggestionWithValue[]> {
     // Autocompletion can leak names of tables and columns.
     if (!await this._granularAccess.canScanData(docSession)) { return []; }
@@ -1479,16 +1486,16 @@ export class ActiveDoc extends EventEmitter {
   /**
    * Check if an ACL formula is valid. If not, will throw an error with an explanation.
    */
-  public async checkAclFormula(docSession: DocSession, text: string): Promise<FormulaProperties> {
+  public async checkAclFormula(docSession: DocSession, text: string): Promise<PredicateFormulaProperties> {
     // Checks can leak names of tables and columns.
     if (await this._granularAccess.hasNuancedAccess(docSession)) { return {}; }
     await this.waitForInitialization();
     try {
-      const parsedAclFormula = await this._pyCall('parse_acl_formula', text);
+      const parsedAclFormula = await this._pyCall('parse_predicate_formula', text);
-      compileAclFormula(parsedAclFormula);
+      compilePredicateFormula(parsedAclFormula);
       // TODO We also need to check the validity of attributes, and of tables and columns
       // mentioned in resources and userAttribute rules.
-      return getFormulaProperties(parsedAclFormula);
+      return getPredicateFormulaProperties(parsedAclFormula);
     } catch (e) {
       e.message = e.message?.replace('[Sandbox] ', '');
       throw e;

app/server/lib/GranularAccess.ts

@@ -26,16 +26,15 @@ import { UserOverride } from 'app/common/DocListAPI';
 import { DocUsageSummary, FilteredDocUsageSummary } from 'app/common/DocUsage';
 import { normalizeEmail } from 'app/common/emails';
 import { ErrorWithCode } from 'app/common/ErrorWithCode';
-import { AclMatchInput, InfoEditor, InfoView } from 'app/common/GranularAccessClause';
+import { InfoEditor, InfoView, UserInfo } from 'app/common/GranularAccessClause';
-import { UserInfo } from 'app/common/GranularAccessClause';
 import * as gristTypes from 'app/common/gristTypes';
 import { getSetMapValue, isNonNullish, pruneArray } from 'app/common/gutil';
+import { compilePredicateFormula, EmptyRecordView, PredicateFormulaInput } from 'app/common/PredicateFormula';
 import { MetaRowRecord, SingleCell } from 'app/common/TableData';
 import { canEdit, canView, isValidRole, Role } from 'app/common/roles';
 import { FullUser, UserAccessData } from 'app/common/UserAPI';
 import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
 import { GristObjCode } from 'app/plugin/GristData';
-import { compileAclFormula } from 'app/server/lib/ACLFormula';
 import { DocClients } from 'app/server/lib/DocClients';
 import { getDocSessionAccess, getDocSessionAltSessionId, getDocSessionShare,
          getDocSessionUser, OptDocSession } from 'app/server/lib/DocSession';
@@ -344,7 +343,7 @@ export class GranularAccess implements GranularAccessForBundle {
    * Represent fields from the session in an input object for ACL rules.
    * Just one field currently, "user".
    */
-  public async inputs(docSession: OptDocSession): Promise<AclMatchInput> {
+  public async inputs(docSession: OptDocSession): Promise<PredicateFormulaInput> {
     return {
       user: await this._getUser(docSession),
       docId: this._docId
@@ -401,7 +400,7 @@ export class GranularAccess implements GranularAccessForBundle {
     }
     const rec = new RecordView(rows, 0);
     if (!hasExceptionalAccess) {
-      const input: AclMatchInput = {...await this.inputs(docSession), rec, newRec: rec};
+      const input: PredicateFormulaInput = {...await this.inputs(docSession), rec, newRec: rec};
       const rowPermInfo = new PermissionInfo(this._ruler.ruleCollection, input);
       const rowAccess = rowPermInfo.getTableAccess(cell.tableId).perms.read;
       if (rowAccess === 'deny') { fail(); }
@@ -560,7 +559,7 @@ export class GranularAccess implements GranularAccessForBundle {
 
       // Use the post-actions data to process the rules collection, and throw error if that fails.
       const ruleCollection = new ACLRuleCollection();
-      await ruleCollection.update(tmpDocData, {log, compile: compileAclFormula});
+      await ruleCollection.update(tmpDocData, {log, compile: compilePredicateFormula});
       if (ruleCollection.ruleError) {
         throw new ApiError(ruleCollection.ruleError.message, 400);
       }
@@ -1664,7 +1663,7 @@ export class GranularAccess implements GranularAccessForBundle {
 
     const rec = new RecordView(rowsRec, undefined);
     const newRec = new RecordView(rowsNewRec, undefined);
-    const input: AclMatchInput = {...await this.inputs(docSession), rec, newRec};
+    const input: PredicateFormulaInput = {...await this.inputs(docSession), rec, newRec};
 
     const [, tableId, , colValues] = action;
     let filteredColValues: ColValues | BulkColValues | undefined | null = null;
@@ -1746,7 +1745,7 @@ export class GranularAccess implements GranularAccessForBundle {
                                   colId?: string): Promise<number[]> {
     const ruler = await this._getRuler(cursor);
     const rec = new RecordView(data, undefined);
-    const input: AclMatchInput = {...await this.inputs(cursor.docSession), rec};
+    const input: PredicateFormulaInput = {...await this.inputs(cursor.docSession), rec};
 
     const [, tableId, rowIds] = data;
     const toRemove: number[] = [];
@@ -2561,7 +2560,7 @@ export class GranularAccess implements GranularAccessForBundle {
         }
       }
       const rec = rows ? new RecordView(rows, 0) : undefined;
-      const input: AclMatchInput = {...inputs, rec, newRec: rec};
+      const input: PredicateFormulaInput = {...inputs, rec, newRec: rec};
       const rowPermInfo = new PermissionInfo(ruler.ruleCollection, input);
       const rowAccess = rowPermInfo.getTableAccess(cell.tableId).perms.read;
       if (rowAccess === 'deny') { return false; }
@@ -2635,7 +2634,11 @@ export class Ruler {
    * Update granular access from DocData.
    */
   public async update(docData: DocData) {
-    await this.ruleCollection.update(docData, {log, compile: compileAclFormula, enrichRulesForImplementation: true});
+    await this.ruleCollection.update(docData, {
+      log,
+      compile: compilePredicateFormula,
+      enrichRulesForImplementation: true,
+    });
 
     // Also clear the per-docSession cache of rule evaluations.
     this.clearCache();
@@ -2652,7 +2655,7 @@ export class Ruler {
 
 export interface RulerOwner {
   getUser(docSession: OptDocSession): Promise<UserInfo>;
-  inputs(docSession: OptDocSession): Promise<AclMatchInput>;
+  inputs(docSession: OptDocSession): Promise<PredicateFormulaInput>;
 }
 
 /**
@@ -2762,11 +2765,6 @@ class RecordEditor implements InfoEditor {
   }
 }
 
-class EmptyRecordView implements InfoView {
-  public get(colId: string): CellValue { return null; }
-  public toJSON() { return {}; }
-}
-
 /**
  * Cache information about user attributes.
  */
@@ -2840,7 +2838,7 @@ class CellAccessHelper {
   private _tableAccess: Map<string, boolean> = new Map();
   private _rowPermInfo: Map<string, Map<number, PermissionInfo>> = new Map();
   private _rows: Map<string, TableDataAction> = new Map();
-  private _inputs!: AclMatchInput;
+  private _inputs!: PredicateFormulaInput;
 
   constructor(
     private _granular: GranularAccess,
@@ -2864,7 +2862,7 @@ class CellAccessHelper {
         for(const [idx, rowId] of rows[2].entries()) {
           if (rowIds.has(rowId) === false) { continue; }
           const rec = new RecordView(rows, idx);
-          const input: AclMatchInput = {...this._inputs, rec, newRec: rec};
+          const input: PredicateFormulaInput = {...this._inputs, rec, newRec: rec};
           const rowPermInfo = new PermissionInfo(this._ruler.ruleCollection, input);
           if (!this._rowPermInfo.has(tableId)) {
             this._rowPermInfo.set(tableId, new Map());

app/server/lib/PermissionInfo.ts

@@ -3,7 +3,8 @@ import { ALL_PERMISSION_PROPS, emptyPermissionSet,
          MixedPermissionSet, PartialPermissionSet, PermissionSet, TablePermissionSet,
          toMixed } from 'app/common/ACLPermissions';
 import { ACLRuleCollection } from 'app/common/ACLRuleCollection';
-import { AclMatchInput, RuleSet, UserInfo } from 'app/common/GranularAccessClause';
+import { RuleSet, UserInfo } from 'app/common/GranularAccessClause';
+import { PredicateFormulaInput } from 'app/common/PredicateFormula';
 import { getSetMapValue } from 'app/common/gutil';
 import log from 'app/server/lib/log';
 import { mapValues } from 'lodash';
@@ -59,7 +60,7 @@ abstract class RuleInfo<MixedT extends TableT, TableT> {
 
   // Construct a RuleInfo for a particular input, which is a combination of user and
   // optionally a record.
-  constructor(protected _acls: ACLRuleCollection, protected _input: AclMatchInput) {}
+  constructor(protected _acls: ACLRuleCollection, protected _input: PredicateFormulaInput) {}
 
   public getColumnAspect(tableId: string, colId: string): MixedT {
     const ruleSet: RuleSet|undefined = this._acls.getColumnRuleSet(tableId, colId);
@@ -80,7 +81,7 @@ abstract class RuleInfo<MixedT extends TableT, TableT> {
   }
 
   public getUser(): UserInfo {
-    return this._input.user;
+    return this._input.user!;
   }
 
   protected abstract _processRule(ruleSet: RuleSet, defaultAccess?: () => MixedT): MixedT;
@@ -205,7 +206,7 @@ export class PermissionInfo extends RuleInfo<MixedPermissionSet, TablePermission
  * included, the result may include permission values like 'allowSome', 'denySome', or 'mixed' (for
  * rules with memo).
  */
-function evaluateRule(ruleSet: RuleSet, input: AclMatchInput): PartialPermissionSet {
+function evaluateRule(ruleSet: RuleSet, input: PredicateFormulaInput): PartialPermissionSet {
   let pset: PartialPermissionSet = emptyPermissionSet();
   for (const rule of ruleSet.body) {
     try {
@@ -268,7 +269,7 @@ function evaluateRule(ruleSet: RuleSet, input: AclMatchInput): PartialPermission
  * If a rule has a memo, and passes, add that memo for all permissions it denies.
  * If a rule has a memo, and fails, add that memo for all permissions it allows.
  */
-function extractMemos(ruleSet: RuleSet, input: AclMatchInput): MemoSet {
+function extractMemos(ruleSet: RuleSet, input: PredicateFormulaInput): MemoSet {
   const pset = emptyMemoSet();
   for (const rule of ruleSet.body) {
     try {

sandbox/grist/acl.py

@@ -5,7 +5,8 @@
 import json
 import logging
 
-from acl_formula import parse_acl_grist_entities, parse_acl_formula_json
+from acl_formula import parse_acl_grist_entities
+from predicate_formula import parse_predicate_formula_json
 import action_obj
 import textbuilder
 
@@ -130,7 +131,7 @@ def prepare_acl_col_renames(docmodel, useractions, col_renames_dict):
       replacer = textbuilder.Replacer(textbuilder.Text(formula), patches)
       txt = replacer.get_text()
       rule_updates.append((rule_rec, {'aclFormula': txt,
-                                      'aclFormulaParsed': parse_acl_formula_json(txt)}))
+                                      'aclFormulaParsed': parse_predicate_formula_json(txt)}))
 
   def do_renames():
     useractions.doBulkUpdateFromPairs('_grist_ACLResources', resource_updates)

sandbox/grist/acl_formula.py

@@ -1,63 +1,19 @@
 import ast
-import io
-import json
-import tokenize
-from collections import namedtuple
 
 import asttokens
-import six
 
-from codebuilder import replace_dollar_attrs
+from predicate_formula import NamedEntity, parse_predicate_formula_json, TreeConverter
 
-def parse_acl_formula(acl_formula):
+def parse_acl_formulas(col_values):
   """
-  Parse an ACL formula expression into a parse tree that we can interpret in JS, e.g.
+  Populates `aclFormulaParsed` by parsing `aclFormula` for all `col_values`.
-  "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']".
-
-  The idea is to support enough to express ACL rules flexibly, but we don't need to support too
-  much, since rules should be reasonably simple.
-
-  The returned tree has the form [NODE_TYPE, arguments...], with these NODE_TYPEs supported:
-    And|Or                  ...values
-    Add|Sub|Mult|Div|Mod    left, right
-    Not                     operand
-    Eq|NotEq|Lt|LtE|Gt|GtE  left, right
-    Is|IsNot|In|NotIn       left, right
-    List                    ...elements
-    Const                   value (number, string, bool)
-    Name                    name (string)
-    Attr                    node, attr_name
-    Comment                 node, comment
   """
-  if isinstance(acl_formula, six.binary_type):
+  if 'aclFormula' not in col_values:
-    acl_formula = acl_formula.decode('utf8')
+    return
-  try:
-    acl_formula = replace_dollar_attrs(acl_formula)
-    tree = ast.parse(acl_formula, mode='eval')
-    result = _TreeConverter().visit(tree)
-    for part in tokenize.generate_tokens(io.StringIO(acl_formula).readline):
-      if part[0] == tokenize.COMMENT and part[1].startswith('#'):
-        result = ['Comment', result, part[1][1:].strip()]
-        break
-    return result
-  except SyntaxError as err:
-    # In case of an error, include line and offset.
-    raise SyntaxError("%s on line %s col %s" % (err.args[0], err.lineno, err.offset))
 
-
+  col_values['aclFormulaParsed'] = [parse_predicate_formula_json(v)
-def parse_acl_formula_json(acl_formula):
+                                    for v
-  """
+                                    in col_values['aclFormula']]
-  As parse_acl_formula(), but stringifies the result, and converts empty string to empty string.
-  """
-  return json.dumps(parse_acl_formula(acl_formula)) if acl_formula else ""
-
-
-# Entities encountered in ACL formulas, which may get renamed.
-#   type : 'recCol'|'userAttr'|'userAttrCol',
-#   start_pos: number,        # start position of the token in the code.
-#   name: string,             # the name that may be updated by a rename.
-#   extra: string|None,       # name of userAttr in case of userAttrCol; otherwise None.
-NamedEntity = namedtuple('NamedEntity', ('type', 'start_pos', 'name', 'extra'))
 
 def parse_acl_grist_entities(acl_formula):
   """
@@ -72,69 +28,7 @@ def parse_acl_grist_entities(acl_formula):
   except SyntaxError as err:
     return []
 
-
+class _EntityCollector(TreeConverter):
-named_constants = {
-  'True': True,
-  'False': False,
-  'None': None,
-}
-
-class _TreeConverter(ast.NodeVisitor):
-  # AST nodes are documented here: https://docs.python.org/2/library/ast.html#abstract-grammar
-  # pylint:disable=no-self-use
-
-  def visit_Expression(self, node):
-    return self.visit(node.body)
-
-  def visit_BoolOp(self, node):
-    return [node.op.__class__.__name__] + [self.visit(v) for v in node.values]
-
-  def visit_BinOp(self, node):
-    if not isinstance(node.op, (ast.Add, ast.Sub, ast.Mult, ast.Div, ast.Mod)):
-      return self.generic_visit(node)
-    return [node.op.__class__.__name__, self.visit(node.left), self.visit(node.right)]
-
-  def visit_UnaryOp(self, node):
-    if not isinstance(node.op, (ast.Not)):
-      return self.generic_visit(node)
-    return [node.op.__class__.__name__, self.visit(node.operand)]
-
-  def visit_Compare(self, node):
-    # We don't try to support chained comparisons like "1 < 2 < 3" (though it wouldn't be hard).
-    if len(node.ops) != 1 or len(node.comparators) != 1:
-      raise ValueError("Can't use chained comparisons")
-    return [node.ops[0].__class__.__name__, self.visit(node.left), self.visit(node.comparators[0])]
-
-  def visit_Name(self, node):
-    if node.id in named_constants:
-      return ["Const", named_constants[node.id]]
-    return ["Name", node.id]
-
-  def visit_Constant(self, node):
-    return ["Const", node.value]
-
-  visit_NameConstant = visit_Constant
-
-  def visit_Attribute(self, node):
-    return ["Attr", self.visit(node.value), node.attr]
-
-  def visit_Num(self, node):
-    return ["Const", node.n]
-
-  def visit_Str(self, node):
-    return ["Const", node.s]
-
-  def visit_List(self, node):
-    return ["List"] + [self.visit(e) for e in node.elts]
-
-  def visit_Tuple(self, node):
-    return self.visit_List(node)    # We don't distinguish tuples and lists
-
-  def generic_visit(self, node):
-    raise ValueError("Unsupported syntax at %s:%s" % (node.lineno, node.col_offset + 1))
-
-
-class _EntityCollector(_TreeConverter):
   def __init__(self):
     self.entities = []    # NamedEntity list
 

sandbox/grist/dropdown_condition.py

@@ -0,0 +1,43 @@
+import json
+import logging
+
+from predicate_formula import parse_predicate_formula_json
+
+log = logging.getLogger(__name__)
+
+def parse_dropdown_conditions(col_values):
+  """
+  Parses any unparsed dropdown conditions in `col_values`.
+  """
+  if 'widgetOptions' not in col_values:
+    return
+
+  col_values['widgetOptions'] = [parse_dropdown_condition(widget_options_json)
+                                 for widget_options_json
+                                 in col_values['widgetOptions']]
+
+def parse_dropdown_condition(widget_options_json):
+  """
+  Parses `dropdownCondition.text` in `widget_options_json` and stores the parsed
+  representation in `dropdownCondition.parsed`.
+
+  If `dropdownCondition.parsed` is already set, parsing is skipped (as an optimization).
+  Clients are responsible for including just `dropdownCondition.text` when creating new
+  (or updating existing) dropdown conditions.
+
+  Returns an updated copy of `widget_options_json` or the original widget_options_json
+  if parsing was skipped.
+  """
+  try:
+    widget_options = json.loads(widget_options_json)
+    if 'dropdownCondition' not in widget_options:
+      return widget_options_json
+
+    dropdown_condition = widget_options['dropdownCondition']
+    if 'parsed' in dropdown_condition:
+      return widget_options_json
+
+    dropdown_condition['parsed'] = parse_predicate_formula_json(dropdown_condition['text'])
+    return json.dumps(widget_options)
+  except (TypeError, ValueError):
+    return widget_options_json

sandbox/grist/main.py

@@ -23,7 +23,7 @@ import migrations
 import schema
 import useractions
 import objtypes
-from acl_formula import parse_acl_formula
+from predicate_formula import parse_predicate_formula
 from sandbox import get_default_sandbox
 from imports.register import register_import_parsers
 
@@ -174,7 +174,7 @@ def run(sandbox):
   def get_timings():
     return eng._timing.get()
 
-  export(parse_acl_formula)
+  export(parse_predicate_formula)
   export(eng.load_empty)
   export(eng.load_done)
 

sandbox/grist/predicate_formula.py

@@ -0,0 +1,118 @@
+import ast
+import io
+import json
+import tokenize
+from collections import namedtuple
+
+import six
+
+from codebuilder import replace_dollar_attrs
+
+# Entities encountered in predicate formulas, which may get renamed.
+#   type : 'recCol'|'userAttr'|'userAttrCol',
+#   start_pos: number,        # start position of the token in the code.
+#   name: string,             # the name that may be updated by a rename.
+#   extra: string|None,       # name of userAttr in case of userAttrCol; otherwise None.
+NamedEntity = namedtuple('NamedEntity', ('type', 'start_pos', 'name', 'extra'))
+
+def parse_predicate_formula(formula):
+  """
+  Parse a predicate formula expression into a parse tree that we can interpret in JS, e.g.
+  "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']".
+
+  The idea is to support enough to express ACL rules and dropdown conditions flexibly, but we
+  don't need to support too much, since expressions should be reasonably simple.
+
+  The returned tree has the form [NODE_TYPE, arguments...], with these NODE_TYPEs supported:
+    And|Or                  ...values
+    Add|Sub|Mult|Div|Mod    left, right
+    Not                     operand
+    Eq|NotEq|Lt|LtE|Gt|GtE  left, right
+    Is|IsNot|In|NotIn       left, right
+    List                    ...elements
+    Const                   value (number, string, bool)
+    Name                    name (string)
+    Attr                    node, attr_name
+    Comment                 node, comment
+  """
+  if isinstance(formula, six.binary_type):
+    formula = formula.decode('utf8')
+  try:
+    formula = replace_dollar_attrs(formula)
+    tree = ast.parse(formula, mode='eval')
+    result = TreeConverter().visit(tree)
+    for part in tokenize.generate_tokens(io.StringIO(formula).readline):
+      if part[0] == tokenize.COMMENT and part[1].startswith('#'):
+        result = ['Comment', result, part[1][1:].strip()]
+        break
+    return result
+  except SyntaxError as err:
+    # In case of an error, include line and offset.
+    raise SyntaxError("%s on line %s col %s" % (err.args[0], err.lineno, err.offset))
+
+def parse_predicate_formula_json(formula):
+  """
+  As parse_predicate_formula(), but stringifies the result, and converts falsy
+  values to empty string.
+  """
+  return json.dumps(parse_predicate_formula(formula)) if formula else ""
+
+named_constants = {
+  'True': True,
+  'False': False,
+  'None': None,
+}
+
+class TreeConverter(ast.NodeVisitor):
+  # AST nodes are documented here: https://docs.python.org/2/library/ast.html#abstract-grammar
+  # pylint:disable=no-self-use
+
+  def visit_Expression(self, node):
+    return self.visit(node.body)
+
+  def visit_BoolOp(self, node):
+    return [node.op.__class__.__name__] + [self.visit(v) for v in node.values]
+
+  def visit_BinOp(self, node):
+    if not isinstance(node.op, (ast.Add, ast.Sub, ast.Mult, ast.Div, ast.Mod)):
+      return self.generic_visit(node)
+    return [node.op.__class__.__name__, self.visit(node.left), self.visit(node.right)]
+
+  def visit_UnaryOp(self, node):
+    if not isinstance(node.op, (ast.Not)):
+      return self.generic_visit(node)
+    return [node.op.__class__.__name__, self.visit(node.operand)]
+
+  def visit_Compare(self, node):
+    # We don't try to support chained comparisons like "1 < 2 < 3" (though it wouldn't be hard).
+    if len(node.ops) != 1 or len(node.comparators) != 1:
+      raise ValueError("Can't use chained comparisons")
+    return [node.ops[0].__class__.__name__, self.visit(node.left), self.visit(node.comparators[0])]
+
+  def visit_Name(self, node):
+    if node.id in named_constants:
+      return ["Const", named_constants[node.id]]
+    return ["Name", node.id]
+
+  def visit_Constant(self, node):
+    return ["Const", node.value]
+
+  visit_NameConstant = visit_Constant
+
+  def visit_Attribute(self, node):
+    return ["Attr", self.visit(node.value), node.attr]
+
+  def visit_Num(self, node):
+    return ["Const", node.n]
+
+  def visit_Str(self, node):
+    return ["Const", node.s]
+
+  def visit_List(self, node):
+    return ["List"] + [self.visit(e) for e in node.elts]
+
+  def visit_Tuple(self, node):
+    return self.visit_List(node)    # We don't distinguish tuples and lists
+
+  def generic_visit(self, node):
+    raise ValueError("Unsupported syntax at %s:%s" % (node.lineno, node.col_offset + 1))

sandbox/grist/test_acl_formula.py

@@ -1,151 +1,8 @@
 # -*- coding: utf-8 -*-
 # pylint:disable=line-too-long
 
-import unittest
-from acl_formula import parse_acl_formula
 import test_engine
 
-
-class TestACLFormula(unittest.TestCase):
-  def test_basic(self):
-    # Test a few basic formulas and structures, hitting everything we expect to support
-    self.assertEqual(parse_acl_formula(
-      "user.Email == 'X@'"),
-      ["Eq", ["Attr", ["Name", "user"], "Email"],
-        ["Const", "X@"]])
-
-    self.assertEqual(parse_acl_formula(
-      "user.Role in ('editors', 'owners')"),
-      ["In", ["Attr", ["Name", "user"], "Role"],
-             ["List", ["Const", "editors"], ["Const", "owners"]]])
-
-    self.assertEqual(parse_acl_formula(
-      "user.Role not in ('editors', 'owners')"),
-      ["NotIn", ["Attr", ["Name", "user"], "Role"],
-                ["List", ["Const", "editors"], ["Const", "owners"]]])
-
-    self.assertEqual(parse_acl_formula(
-      "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']"),
-      ['And',
-        ['Eq', ['Attr', ['Name', 'rec'], 'office'], ['Const', 'Seattle']],
-        ['In',
-         ['Attr', ['Name', 'user'], 'email'],
-         ['List', ['Const', 'sally@'], ['Const', 'xie@']]
-        ]])
-
-    self.assertEqual(parse_acl_formula(
-      "$office == 'Seattle' and user.email in ['sally@', 'xie@']"),
-      ['And',
-        ['Eq', ['Attr', ['Name', 'rec'], 'office'], ['Const', 'Seattle']],
-        ['In',
-         ['Attr', ['Name', 'user'], 'email'],
-         ['List', ['Const', 'sally@'], ['Const', 'xie@']]
-        ]])
-
-    self.assertEqual(parse_acl_formula(
-      "user.IsAdmin or rec.assigned is None or (not newRec.HasDuplicates and rec.StatusIndex <= newRec.StatusIndex)"),
-      ['Or',
-        ['Attr', ['Name', 'user'], 'IsAdmin'],
-        ['Is', ['Attr', ['Name', 'rec'], 'assigned'], ['Const', None]],
-        ['And',
-          ['Not', ['Attr', ['Name', 'newRec'], 'HasDuplicates']],
-          ['LtE', ['Attr', ['Name', 'rec'], 'StatusIndex'], ['Attr', ['Name', 'newRec'], 'StatusIndex']]
-        ]
-      ])
-
-    self.assertEqual(parse_acl_formula(
-      "user.IsAdmin or $assigned is None or (not newRec.HasDuplicates and $StatusIndex <= newRec.StatusIndex)"),
-      ['Or',
-        ['Attr', ['Name', 'user'], 'IsAdmin'],
-        ['Is', ['Attr', ['Name', 'rec'], 'assigned'], ['Const', None]],
-        ['And',
-          ['Not', ['Attr', ['Name', 'newRec'], 'HasDuplicates']],
-          ['LtE', ['Attr', ['Name', 'rec'], 'StatusIndex'], ['Attr', ['Name', 'newRec'], 'StatusIndex']]
-        ]
-      ])
-
-    self.assertEqual(parse_acl_formula(
-      "r.A <= n.A + 1 or r.A >= n.A - 1 or r.B < n.B * 2.5 or r.B > n.B / 2.5 or r.C % 2 != 0"),
-      ['Or',
-        ['LtE',
-          ['Attr', ['Name', 'r'], 'A'],
-          ['Add', ['Attr', ['Name', 'n'], 'A'], ['Const', 1]]],
-        ['GtE',
-          ['Attr', ['Name', 'r'], 'A'],
-          ['Sub', ['Attr', ['Name', 'n'], 'A'], ['Const', 1]]],
-        ['Lt',
-          ['Attr', ['Name', 'r'], 'B'],
-          ['Mult', ['Attr', ['Name', 'n'], 'B'], ['Const', 2.5]]],
-        ['Gt',
-          ['Attr', ['Name', 'r'], 'B'],
-          ['Div', ['Attr', ['Name', 'n'], 'B'], ['Const', 2.5]]],
-        ['NotEq',
-          ['Mod', ['Attr', ['Name', 'r'], 'C'], ['Const', 2]],
-          ['Const', 0]]
-      ])
-
-    self.assertEqual(parse_acl_formula(
-      "rec.A is True or rec.A is not False"),
-      ['Or',
-        ['Is', ['Attr', ['Name', 'rec'], 'A'], ['Const', True]],
-        ['IsNot', ['Attr', ['Name', 'rec'], 'A'], ['Const', False]]
-      ])
-
-    self.assertEqual(parse_acl_formula(
-      "$A is True or $A is not False"),
-      ['Or',
-        ['Is', ['Attr', ['Name', 'rec'], 'A'], ['Const', True]],
-        ['IsNot', ['Attr', ['Name', 'rec'], 'A'], ['Const', False]]
-      ])
-
-    self.assertEqual(parse_acl_formula(
-      "user.Office.City == 'Seattle' and user.Status.IsActive"),
-      ['And',
-        ['Eq',
-          ['Attr', ['Attr', ['Name', 'user'], 'Office'], 'City'],
-          ['Const', 'Seattle']],
-        ['Attr', ['Attr', ['Name', 'user'], 'Status'], 'IsActive']
-      ])
-
-    self.assertEqual(parse_acl_formula(
-      "True # Comment!  "),
-      ['Comment', ['Const', True], 'Comment!'])
-
-    self.assertEqual(parse_acl_formula(
-      "\"#x\" == \" # Not a comment \"#Comment!"),
-      ['Comment',
-       ['Eq', ['Const', '#x'], ['Const', ' # Not a comment ']],
-       'Comment!'
-      ])
-
-    self.assertEqual(parse_acl_formula(
-      "# Allow owners\nuser.Access == 'owners' # ignored\n# comment ignored"),
-      ['Comment',
-       ['Eq', ['Attr', ['Name', 'user'], 'Access'], ['Const', 'owners']],
-       'Allow owners'
-      ])
-
-  def test_unsupported(self):
-    # Test a few constructs we expect to fail
-    # Not an expression
-    self.assertRaises(SyntaxError, parse_acl_formula, "return 1")
-    self.assertRaises(SyntaxError, parse_acl_formula, "def foo(): pass")
-
-    # Unsupported node type
-    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "max(rec)")
-    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "user.id in {1, 2, 3}")
-    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 if user.IsAnon else 2")
-
-    # Unsupported operation
-    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 | 2")
-    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 << 2")
-    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "~test")
-
-    # Syntax error
-    self.assertRaises(SyntaxError, parse_acl_formula, "[(]")
-    self.assertRaises(SyntaxError, parse_acl_formula, "user.id in (1,2))")
-    self.assertRaisesRegex(SyntaxError, r'invalid syntax on line 1 col 9', parse_acl_formula, "foo and !bar")
-
 class TestACLFormulaUserActions(test_engine.EngineTestCase):
   def test_acl_actions(self):
     # Adding or updating ACLRules automatically includes aclFormula compilation.

sandbox/grist/test_dropdown_condition.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+# pylint:disable=line-too-long
+import json
+
+import test_engine
+
+class TestDropdownConditionUserActions(test_engine.EngineTestCase):
+  def test_dropdown_condition_col_actions(self):
+    self.apply_user_action(['AddTable', 'Table1', [
+      {'id': 'A', 'type': 'Text'},
+      {'id': 'B', 'type': 'Text'},
+      {'id': 'C', 'type': 'Text'},
+    ]])
+
+    # Check that setting dropdownCondition.text automatically sets a parsed version.
+    out_actions = self.apply_user_action(['UpdateRecord', '_grist_Tables_column', 1, {
+        "widgetOptions": json.dumps({
+          "dropdownCondition": {
+            "text": 'choice.Role == "Manager"',
+          },
+        }),
+    }])
+    self.assertPartialOutActions(out_actions, { "stored": [
+      ["UpdateRecord", "_grist_Tables_column", 1, {
+        "widgetOptions": "{\"dropdownCondition\": {\"text\": "
+          + "\"choice.Role == \\\"Manager\\\"\", \"parsed\": "
+          + "\"[\\\"Eq\\\", [\\\"Attr\\\", [\\\"Name\\\", \\\"choice\\\"], "
+          + "\\\"Role\\\"], [\\\"Const\\\", \\\"Manager\\\"]]\"}}"
+      }]
+    ]})
+    out_actions = self.apply_user_action(['BulkUpdateRecord', '_grist_Tables_column', [2, 3], {
+      "widgetOptions": [
+        json.dumps({
+          "dropdownCondition": {
+            "text": 'choice == "Manager"',
+          },
+        }),
+        json.dumps({
+          "dropdownCondition": {
+            "text": '$Role == "Manager"',
+          },
+        }),
+      ],
+    }])
+    self.assertPartialOutActions(out_actions, { "stored": [
+      ["BulkUpdateRecord", "_grist_Tables_column", [2, 3], {
+        "widgetOptions": [
+          "{\"dropdownCondition\": {\"text\": \"choice == "
+            + "\\\"Manager\\\"\", \"parsed\": \"[\\\"Eq\\\", "
+            + "[\\\"Name\\\", \\\"choice\\\"], [\\\"Const\\\", \\\"Manager\\\"]]\"}}",
+          "{\"dropdownCondition\": {\"text\": \"$Role == "
+            + "\\\"Manager\\\"\", \"parsed\": \"[\\\"Eq\\\", "
+            + "[\\\"Attr\\\", [\\\"Name\\\", \\\"rec\\\"], \\\"Role\\\"], "
+            + "[\\\"Const\\\", \\\"Manager\\\"]]\"}}",
+        ]
+      }]
+    ]})
+
+  def test_dropdown_condition_field_actions(self):
+    self.apply_user_action(['AddTable', 'Table1', [
+      {'id': 'A', 'type': 'Text'},
+      {'id': 'B', 'type': 'Text'},
+      {'id': 'C', 'type': 'Text'},
+    ]])
+
+    # Check that setting dropdownCondition.text automatically sets a parsed version.
+    out_actions = self.apply_user_action(['UpdateRecord', '_grist_Views_section_field', 1, {
+        "widgetOptions": json.dumps({
+          "dropdownCondition": {
+            "text": 'choice.Role == "Manager"',
+          },
+        }),
+    }])
+    self.assertPartialOutActions(out_actions, { "stored": [
+      ["UpdateRecord", "_grist_Views_section_field", 1, {
+        "widgetOptions": "{\"dropdownCondition\": {\"text\": "
+          + "\"choice.Role == \\\"Manager\\\"\", \"parsed\": "
+          + "\"[\\\"Eq\\\", [\\\"Attr\\\", [\\\"Name\\\", \\\"choice\\\"], "
+          + "\\\"Role\\\"], [\\\"Const\\\", \\\"Manager\\\"]]\"}}"
+      }]
+    ]})
+    out_actions = self.apply_user_action(['BulkUpdateRecord', '_grist_Views_section_field', [2, 3], {
+      "widgetOptions": [
+        json.dumps({
+          "dropdownCondition": {
+            "text": 'choice == "Manager"',
+          },
+        }),
+        json.dumps({
+          "dropdownCondition": {
+            "text": '$Role == "Manager"',
+          },
+        }),
+      ],
+    }])
+    self.assertPartialOutActions(out_actions, { "stored": [
+      ["BulkUpdateRecord", "_grist_Views_section_field", [2, 3], {
+        "widgetOptions": [
+          "{\"dropdownCondition\": {\"text\": \"choice == "
+            + "\\\"Manager\\\"\", \"parsed\": \"[\\\"Eq\\\", "
+            + "[\\\"Name\\\", \\\"choice\\\"], [\\\"Const\\\", \\\"Manager\\\"]]\"}}",
+          "{\"dropdownCondition\": {\"text\": \"$Role == "
+            + "\\\"Manager\\\"\", \"parsed\": \"[\\\"Eq\\\", "
+            + "[\\\"Attr\\\", [\\\"Name\\\", \\\"rec\\\"], \\\"Role\\\"], "
+            + "[\\\"Const\\\", \\\"Manager\\\"]]\"}}",
+        ]
+      }]
+    ]})

sandbox/grist/test_predicate_formula.py

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+# pylint:disable=line-too-long
+
+import unittest
+from predicate_formula import parse_predicate_formula
+
+class TestPredicateFormula(unittest.TestCase):
+  def test_basic(self):
+    # Test a few basic formulas and structures, hitting everything we expect to support
+    # in ACL formulas and dropdown conditions.
+    self.assertEqual(parse_predicate_formula(
+      "user.Email == 'X@'"),
+      ["Eq", ["Attr", ["Name", "user"], "Email"],
+        ["Const", "X@"]])
+
+    self.assertEqual(parse_predicate_formula(
+      "user.Role in ('editors', 'owners')"),
+      ["In", ["Attr", ["Name", "user"], "Role"],
+             ["List", ["Const", "editors"], ["Const", "owners"]]])
+
+    self.assertEqual(parse_predicate_formula(
+      "user.Role not in ('editors', 'owners')"),
+      ["NotIn", ["Attr", ["Name", "user"], "Role"],
+                ["List", ["Const", "editors"], ["Const", "owners"]]])
+
+    self.assertEqual(parse_predicate_formula(
+      "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']"),
+      ['And',
+        ['Eq', ['Attr', ['Name', 'rec'], 'office'], ['Const', 'Seattle']],
+        ['In',
+         ['Attr', ['Name', 'user'], 'email'],
+         ['List', ['Const', 'sally@'], ['Const', 'xie@']]
+        ]])
+
+    self.assertEqual(parse_predicate_formula(
+      "$office == 'Seattle' and user.email in ['sally@', 'xie@']"),
+      ['And',
+        ['Eq', ['Attr', ['Name', 'rec'], 'office'], ['Const', 'Seattle']],
+        ['In',
+         ['Attr', ['Name', 'user'], 'email'],
+         ['List', ['Const', 'sally@'], ['Const', 'xie@']]
+        ]])
+
+    self.assertEqual(parse_predicate_formula(
+      "user.IsAdmin or rec.assigned is None or (not newRec.HasDuplicates and rec.StatusIndex <= newRec.StatusIndex)"),
+      ['Or',
+        ['Attr', ['Name', 'user'], 'IsAdmin'],
+        ['Is', ['Attr', ['Name', 'rec'], 'assigned'], ['Const', None]],
+        ['And',
+          ['Not', ['Attr', ['Name', 'newRec'], 'HasDuplicates']],
+          ['LtE', ['Attr', ['Name', 'rec'], 'StatusIndex'], ['Attr', ['Name', 'newRec'], 'StatusIndex']]
+        ]
+      ])
+
+    self.assertEqual(parse_predicate_formula(
+      "user.IsAdmin or $assigned is None or (not newRec.HasDuplicates and $StatusIndex <= newRec.StatusIndex)"),
+      ['Or',
+        ['Attr', ['Name', 'user'], 'IsAdmin'],
+        ['Is', ['Attr', ['Name', 'rec'], 'assigned'], ['Const', None]],
+        ['And',
+          ['Not', ['Attr', ['Name', 'newRec'], 'HasDuplicates']],
+          ['LtE', ['Attr', ['Name', 'rec'], 'StatusIndex'], ['Attr', ['Name', 'newRec'], 'StatusIndex']]
+        ]
+      ])
+
+    self.assertEqual(parse_predicate_formula(
+      "r.A <= n.A + 1 or r.A >= n.A - 1 or r.B < n.B * 2.5 or r.B > n.B / 2.5 or r.C % 2 != 0"),
+      ['Or',
+        ['LtE',
+          ['Attr', ['Name', 'r'], 'A'],
+          ['Add', ['Attr', ['Name', 'n'], 'A'], ['Const', 1]]],
+        ['GtE',
+          ['Attr', ['Name', 'r'], 'A'],
+          ['Sub', ['Attr', ['Name', 'n'], 'A'], ['Const', 1]]],
+        ['Lt',
+          ['Attr', ['Name', 'r'], 'B'],
+          ['Mult', ['Attr', ['Name', 'n'], 'B'], ['Const', 2.5]]],
+        ['Gt',
+          ['Attr', ['Name', 'r'], 'B'],
+          ['Div', ['Attr', ['Name', 'n'], 'B'], ['Const', 2.5]]],
+        ['NotEq',
+          ['Mod', ['Attr', ['Name', 'r'], 'C'], ['Const', 2]],
+          ['Const', 0]]
+      ])
+
+    self.assertEqual(parse_predicate_formula(
+      "rec.A is True or rec.A is not False"),
+      ['Or',
+        ['Is', ['Attr', ['Name', 'rec'], 'A'], ['Const', True]],
+        ['IsNot', ['Attr', ['Name', 'rec'], 'A'], ['Const', False]]
+      ])
+
+    self.assertEqual(parse_predicate_formula(
+      "$A is True or $A is not False"),
+      ['Or',
+        ['Is', ['Attr', ['Name', 'rec'], 'A'], ['Const', True]],
+        ['IsNot', ['Attr', ['Name', 'rec'], 'A'], ['Const', False]]
+      ])
+
+    self.assertEqual(parse_predicate_formula(
+      "user.Office.City == 'Seattle' and user.Status.IsActive"),
+      ['And',
+        ['Eq',
+          ['Attr', ['Attr', ['Name', 'user'], 'Office'], 'City'],
+          ['Const', 'Seattle']],
+        ['Attr', ['Attr', ['Name', 'user'], 'Status'], 'IsActive']
+      ])
+
+    self.assertEqual(parse_predicate_formula(
+      "True # Comment!  "),
+      ['Comment', ['Const', True], 'Comment!'])
+
+    self.assertEqual(parse_predicate_formula(
+      "\"#x\" == \" # Not a comment \"#Comment!"),
+      ['Comment',
+       ['Eq', ['Const', '#x'], ['Const', ' # Not a comment ']],
+       'Comment!'
+      ])
+
+    self.assertEqual(parse_predicate_formula(
+      "# Allow owners\nuser.Access == 'owners' # ignored\n# comment ignored"),
+      ['Comment',
+       ['Eq', ['Attr', ['Name', 'user'], 'Access'], ['Const', 'owners']],
+       'Allow owners'
+      ])
+
+    self.assertEqual(parse_predicate_formula(
+      "choice not in $Categories"),
+      ['NotIn', ['Name', 'choice'], ['Attr', ['Name', 'rec'], 'Categories']])
+
+    self.assertEqual(parse_predicate_formula(
+      "choice.role == \"Manager\""),
+      ['Eq', ['Attr', ['Name', 'choice'], 'role'], ['Const', 'Manager']])
+
+  def test_unsupported(self):
+    # Test a few constructs we expect to fail
+    # Not an expression
+    self.assertRaises(SyntaxError, parse_predicate_formula, "return 1")
+    self.assertRaises(SyntaxError, parse_predicate_formula, "def foo(): pass")
+
+    # Unsupported node type
+    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "max(rec)")
+    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "user.id in {1, 2, 3}")
+    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "1 if user.IsAnon else 2")
+
+    # Unsupported operation
+    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "1 | 2")
+    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "1 << 2")
+    self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "~test")
+
+    # Syntax error
+    self.assertRaises(SyntaxError, parse_predicate_formula, "[(]")
+    self.assertRaises(SyntaxError, parse_predicate_formula, "user.id in (1,2))")
+    self.assertRaisesRegex(SyntaxError, r'invalid syntax on line 1 col 9', parse_predicate_formula, "foo and !bar")

sandbox/grist/useractions.py

@@ -12,7 +12,8 @@ from six.moves import xrange
 import acl
 import depend
 import gencode
-from acl_formula import parse_acl_formula_json
+from acl_formula import parse_acl_formulas
+from dropdown_condition import parse_dropdown_conditions
 import actions
 import column
 import sort_specs
@@ -437,9 +438,7 @@ class UserActions(object):
 
   @override_action('BulkAddRecord', '_grist_ACLRules')
   def _addACLRules(self, table_id, row_ids, col_values):
-    # Automatically populate aclFormulaParsed value by parsing aclFormula.
+    parse_acl_formulas(col_values)
-    if 'aclFormula' in col_values:
-      col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']]
     return self.doBulkAddOrReplace(table_id, row_ids, col_values)
 
   #----------------------------------------
@@ -672,6 +671,7 @@ class UserActions(object):
     #     columns for all summary tables of the same source table).
     # (4) Updates to the source columns of summary group-by columns (including renaming and type
     #     changes) should be copied to those group-by columns.
+    parse_dropdown_conditions(col_values)
 
     # A list of individual (col_rec, values) updates, where values is a per-column dict.
     col_updates = OrderedDict()
@@ -781,11 +781,14 @@ class UserActions(object):
 
     self.doBulkUpdateRecord(table_id, row_ids, col_values)
 
+  @override_action('BulkUpdateRecord', '_grist_Views_section_field')
+  def _updateViewSectionFields(self, table_id, row_ids, col_values):
+    parse_dropdown_conditions(col_values)
+    return self.doBulkUpdateRecord(table_id, row_ids, col_values)
+
   @override_action('BulkUpdateRecord', '_grist_ACLRules')
   def _updateACLRules(self, table_id, row_ids, col_values):
-    # Automatically populate aclFormulaParsed value by parsing aclFormula.
+    parse_acl_formulas(col_values)
-    if 'aclFormula' in col_values:
-      col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']]
     return self.doBulkUpdateRecord(table_id, row_ids, col_values)
 
   def _prepare_formula_renames(self, renames):

test/client/lib/ACIndex.ts

@@ -378,7 +378,12 @@ class BruteForceACIndexImpl<Item extends ACItem> implements ACIndex<Item> {
   public search(searchText: string): ACResults<Item> {
     const cleanedSearchText = searchText.trim().toLowerCase();
     if (!cleanedSearchText) {
-      return {items: this._allItems.slice(0, this._maxResults), highlightFunc: highlightNone, selectIndex: -1};
+      return {
+        items: this._allItems.slice(0, this._maxResults),
+        extraItems: [],
+        highlightFunc: highlightNone,
+        selectIndex: -1,
+      };
     }
 
     const searchWords = cleanedSearchText.split(/\s+/);
@@ -397,7 +402,7 @@ class BruteForceACIndexImpl<Item extends ACItem> implements ACIndex<Item> {
     matches.sort((a, b) => nativeCompare(b[0], a[0]) || nativeCompare(a[1], b[1]));
     const items = matches.slice(0, this._maxResults).map((m) => m[2]);
 
-    return {items, highlightFunc: highlightNone, selectIndex: -1};
+    return {items, extraItems: [], highlightFunc: highlightNone, selectIndex: -1};
   }
 }
 

test/client/lib/SafeBrowser.ts

@@ -3,13 +3,11 @@ import { Disposable } from 'app/client/lib/dispose';
 import { ClientProcess, SafeBrowser } from 'app/client/lib/SafeBrowser';
 import { LocalPlugin } from 'app/common/plugin';
 import { PluginInstance } from 'app/common/PluginInstance';
-import { GristLight } from 'app/common/themes/GristLight';
 import { GristAPI, RPC_GRISTAPI_INTERFACE } from 'app/plugin/GristAPI';
 import { Storage } from 'app/plugin/StorageAPI';
 import { checkers } from 'app/plugin/TypeCheckers';
 import { assert } from 'chai';
 import { Rpc } from 'grain-rpc';
-import { Computed } from 'grainjs';
 import { noop } from 'lodash';
 import { basename } from 'path';
 import * as sinon from 'sinon';
@@ -188,7 +186,6 @@ describe('SafeBrowser', function() {
       untrustedContentOrigin: '',
       mainPath,
       baseLogger: {},
-      theme: Computed.create(null, () => ({appearance: 'light', colors: GristLight})),
     });
     cleanup.push(() => safeBrowser.deactivate());
     pluginInstance.rpc.registerForwarder(mainPath, safeBrowser);

test/client/lib/nameUtils.ts

@@ -1,11 +1,9 @@
-import {checkName} from 'app/client/ui/AccountPage';
+import {checkName} from 'app/client/lib/nameUtils';
 import {assert} from 'chai';
 
-
+describe("nameUtils", function() {
-describe("AccountPage", function() {
   describe("isValidName", function() {
     it("should detect invalid name", function() {
-
       assert.equal(checkName('santa'), true);
       assert.equal(checkName('_santa'), true);
       assert.equal(checkName("O'Neil"), true);

test/client/lib/timeUtils.ts

@@ -1,8 +1,8 @@
-import {getTimeFromNow} from 'app/client/models/HomeModel';
+import {getTimeFromNow} from 'app/client/lib/timeUtils';
 import {assert} from 'chai';
 import moment from 'moment';
 
-describe("HomeModel", function() {
+describe("timeUtils", function() {
   describe("getTimeFromNow", function() {
     it("should give good summary of time that just passed", function() {
       const t = moment().subtract(10, 's');
@@ -18,5 +18,5 @@ describe("HomeModel", function() {
       const t = moment().add(2, 'minutes');
       assert.equal(getTimeFromNow(t.toISOString()), 'in 2 minutes');
     });
-});
+  });
 });

test/nbrowser/DropdownConditionEditor.ts

@@ -0,0 +1,272 @@
+import {assert, driver, Key} from 'mocha-webdriver';
+import * as gu from 'test/nbrowser/gristUtils';
+import {setupTestSuite} from 'test/nbrowser/testUtils';
+
+describe('DropdownConditionEditor', function () {
+  this.timeout(20000);
+  const cleanup = setupTestSuite();
+
+  before(async () => {
+    const session = await gu.session().login();
+    await session.tempDoc(cleanup, 'DropdownCondition.grist');
+    await gu.openColumnPanel();
+  });
+
+  afterEach(() => gu.checkForErrors());
+
+  describe(`in choice columns`, function() {
+    it('creates dropdown conditions', async function() {
+      await gu.getCell(1, 1).click();
+      assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent());
+      await driver.find('.test-field-set-dropdown-condition').click();
+      await gu.sendKeys('c');
+      await gu.waitToPass(async () => {
+        const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText());
+        assert.deepEqual(completions, [
+          'c\nhoice\n ',
+          're\nc\n.Name\n ',
+          're\nc\n.Role\n ',
+          're\nc\n.Supervisor\n ',
+        ]);
+      });
+      await gu.sendKeys('hoice not in $');
+      await gu.waitToPass(async () => {
+        const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText());
+        assert.deepEqual(completions, [
+          '$\nName\n ',
+          '$\nRole\n ',
+          '$\nSupervisor\n ',
+        ]);
+      });
+      await gu.sendKeys('Role', Key.ENTER);
+      await gu.waitForServer();
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition').getText(),
+        'choice not in $Role'
+      );
+
+      // Check that autocomplete values are filtered.
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Supervisor',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+      await gu.getCell(1, 4).click();
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Trainee',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+      await gu.getCell(1, 6).click();
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Trainee',
+        'Supervisor',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+
+      // Change the column type to Choice List and check values are still filtered.
+      await gu.setType('Choice List', {apply: true});
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition').getText(),
+        'choice not in $Role'
+      );
+      await gu.getCell(1, 4).click();
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Trainee',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+    });
+
+    it('removes dropdown conditions', async function() {
+      await driver.find('.test-field-dropdown-condition').click();
+      await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, Key.ENTER);
+      await gu.waitForServer();
+
+      // Check that autocomplete values are no longer filtered.
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Trainee',
+        'Supervisor',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+
+      // Change the column type back to Choice and check values are still no longer filtered.
+      await gu.setType('Choice', {apply: true});
+      assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent());
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Supervisor',
+        'Trainee',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+    });
+
+    it('reports errors', async function() {
+      // Check syntax errors are reported, but not saved.
+      await driver.find('.test-field-set-dropdown-condition').click();
+      await gu.sendKeys('!@#$%^', Key.ENTER);
+      await gu.waitForServer();
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition-error').getText(),
+        'SyntaxError invalid syntax on line 1 col 1'
+      );
+      await gu.reloadDoc();
+      assert.isFalse(await driver.find('.test-field-dropdown-condition-error').isPresent());
+
+      // Check compilation errors are reported and saved.
+      await driver.find('.test-field-set-dropdown-condition').click();
+      await gu.sendKeys('foo', Key.ENTER);
+      await gu.waitForServer();
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition-error').getText(),
+        "Unknown variable 'foo'"
+      );
+      await gu.reloadDoc();
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition-error').getText(),
+        "Unknown variable 'foo'"
+      );
+
+      // Check that the autocomplete dropdown also reports an error.
+      await gu.sendKeys(Key.ENTER);
+      assert.equal(
+        await driver.find('.test-autocomplete-no-items-message').getText(),
+        'Error in dropdown condition'
+      );
+      await gu.sendKeys(Key.ESCAPE);
+    });
+  });
+
+  describe(`in reference columns`, function() {
+    it('creates dropdown conditions', async function() {
+      await gu.getCell(2, 1).click();
+      assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent());
+      await driver.find('.test-field-set-dropdown-condition').click();
+      await gu.sendKeys('choice');
+      await gu.waitToPass(async () => {
+        const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText());
+        assert.deepEqual(completions, [
+          'choice\n ',
+          'choice\n.id\n ',
+          'choice\n.Name\n ',
+          'choice\n.Role\n ',
+          'choice\n.Supervisor\n '
+        ]);
+      });
+      await gu.sendKeys('.Role == "Supervisor" and $Role != "Supervisor"', Key.ENTER);
+      await gu.waitForServer();
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition .ace_line').getAttribute('textContent'),
+        'choice.Role == "Supervisor" and $Role != "Supervisor"\n'
+      );
+
+      // Check that autocomplete values are filtered.
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Pavan Madilyn',
+        'Marie Ziyad',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+      await gu.getCell(2, 4).click();
+      await gu.sendKeys(Key.ENTER);
+      assert.isEmpty(await driver.findAll('.test-autocomplete li', (el) => el.getText()));
+      await gu.sendKeys(Key.ESCAPE);
+      await gu.getCell(2, 6).click();
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Marie Ziyad',
+        'Pavan Madilyn',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+
+      // Change the column type to Reference List and check values are still filtered.
+      await gu.setType('Reference List', {apply: true});
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition .ace_line').getAttribute('textContent'),
+        'choice.Role == "Supervisor" and $Role != "Supervisor"\n'
+      );
+      await gu.getCell(2, 4).click();
+      await gu.sendKeys(Key.ENTER);
+      assert.isEmpty(await driver.findAll('.test-autocomplete li', (el) => el.getText()));
+      await gu.sendKeys(Key.ESCAPE);
+    });
+
+    it('removes dropdown conditions', async function() {
+      await driver.find('.test-field-dropdown-condition').click();
+      await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, Key.ENTER);
+      await gu.waitForServer();
+
+      // Check that autocomplete values are no longer filtered.
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Emma Thamir',
+        'Holger Klyment',
+        'Marie Ziyad',
+        'Olivier Bipin',
+        'Pavan Madilyn',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+
+      // Change the column type back to Reference and check values are still no longer filtered.
+      await gu.setType('Reference', {apply: true});
+      assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent());
+      await gu.sendKeys(Key.ENTER);
+      assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [
+        'Emma Thamir',
+        'Holger Klyment',
+        'Marie Ziyad',
+        'Olivier Bipin',
+        'Pavan Madilyn',
+      ]);
+      await gu.sendKeys(Key.ESCAPE);
+    });
+
+    it('reports errors', async function() {
+      // Check syntax errors are reported, but not saved.
+      await driver.find('.test-field-set-dropdown-condition').click();
+      await gu.sendKeys('!@#$%^', Key.ENTER);
+      await gu.waitForServer();
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition-error').getText(),
+        'SyntaxError invalid syntax on line 1 col 1'
+      );
+      await gu.reloadDoc();
+      assert.isFalse(await driver.find('.test-field-dropdown-condition-error').isPresent());
+
+      // Check compilation errors are reported and saved.
+      await driver.find('.test-field-set-dropdown-condition').click();
+      await gu.sendKeys('foo', Key.ENTER);
+      await gu.waitForServer();
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition-error').getText(),
+        "Unknown variable 'foo'"
+      );
+      await gu.reloadDoc();
+      assert.equal(
+        await driver.find('.test-field-dropdown-condition-error').getText(),
+        "Unknown variable 'foo'"
+      );
+
+      // Check that the autocomplete dropdown also reports an error.
+      await gu.sendKeys(Key.ENTER);
+      assert.equal(
+        await driver.find('.test-autocomplete-no-items-message').getText(),
+        'Error in dropdown condition'
+      );
+      await gu.sendKeys(Key.ESCAPE);
+
+      // Check evaluation errors are also reported in the dropdown.
+      await driver.find('.test-field-dropdown-condition').click();
+      await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, '[] not in 5', Key.ENTER);
+      await gu.waitForServer();
+      await gu.sendKeys(Key.ENTER);
+      assert.equal(
+        await driver.find('.test-autocomplete-no-items-message').getText(),
+        'Error in dropdown condition'
+      );
+      await gu.sendKeys(Key.ESCAPE);
+    });
+  });
+});

test/nbrowser/Importer2.ts

@@ -0,0 +1,818 @@
+/**
+ * Test of the Importer dialog (part 2), for imports inside an open doc.
+ */
+import {DocAPI} from 'app/common/UserAPI';
+import {DocCreationInfo} from 'app/common/DocListAPI';
+import * as _ from 'lodash';
+import {assert, driver, Key} from 'mocha-webdriver';
+import * as gu from 'test/nbrowser/gristUtils';
+import {getColumnMatchingRows, getPreviewDiffCellValues, openSource as openSourceFor,
+        openTableMapping, waitForColumnMapping, waitForDiffPreviewToLoad} from 'test/nbrowser/importerTestUtils';
+import {setupTestSuite} from 'test/nbrowser/testUtils';
+
+describe('Importer2', function() {
+  this.timeout(60000);
+  const cleanup = setupTestSuite();
+  let doc: DocCreationInfo;
+  let api: DocAPI;
+
+  before(async function() {
+    // Log in and import a sample document.
+    const session = await gu.session().teamSite.login();
+    doc = await session.tempDoc(cleanup, 'Hello.grist');
+    api = session.createHomeApi().getDocAPI(doc.id);
+  });
+
+  afterEach(() => gu.checkForErrors());
+
+  it("should import new tables losslessly", async function() {
+    // Import mixed_dates.csv into a new table
+    await gu.importFileDialog('./uploads/mixed_dates.csv');
+    await waitForDiffPreviewToLoad();
+    await driver.find('.test-modal-confirm').click();
+    await gu.waitAppFocus();
+
+    // Import the same file again into the same table
+    await gu.importFileDialog('./uploads/mixed_dates.csv');
+    await driver.findContent('.test-importer-target-existing-table', /Mixed_dates/).click();
+    await waitForDiffPreviewToLoad();
+    await driver.find('.test-modal-confirm').click();
+    await gu.waitAppFocus();
+
+    assert.deepEqual(
+      await gu.getVisibleGridCells({cols: [0], rowNums: _.range(1, 21)}),
+      [
+        // mixed_dates.csv contains 10 dates. The first 9 are YYYY-MM-DD so that's the guessed date format.
+        // The last date '01/02/03' doesn't fit this format.
+        // Since 90% of the values fit the guessed format, the column is guessed to have type Date.
+        // The dates are parsed by DateGuesser which uses moment's strict parsing directly, not parseDate.
+        // So '01/02/03' isn't parsed and remains a string, and the column is imported losslessly,
+        // i.e. converting it back to text yields the original strings in the file unchanged.
+        '2020-03-04',
+        '2020-03-05',
+        '2020-03-06',
+        '2020-03-04',
+        '2020-03-05',
+        '2020-03-06',
+        '2020-03-04',
+        '2020-03-05',
+        '2020-03-06',
+        '01/02/03',
+
+        // When the file is imported again into the same table, things go differently.
+        // The intermediate hidden table goes through the same process and stores '01/02/03' as a string.
+        // But for existing tables we set parseStrings to true when applying the final BulkAddRecord.
+        // So '01/02/03' is parsed by parseDate according to the existing column's date format which gives 2001-02-03.
+        '2020-03-04',
+        '2020-03-05',
+        '2020-03-06',
+        '2020-03-04',
+        '2020-03-05',
+        '2020-03-06',
+        '2020-03-04',
+        '2020-03-05',
+        '2020-03-06',
+        '2001-02-03',
+      ],
+    );
+
+    await gu.undo(2);
+  });
+
+  it("should set widget options for formatted numbers", async function() {
+    // Import formatted_numbers.csv into a new table
+    await gu.importFileDialog('./uploads/formatted_numbers.csv');
+    await waitForDiffPreviewToLoad();
+    await driver.find('.test-modal-confirm').click();
+    await gu.waitAppFocus();
+
+    // Numbers appear formatted as in the CSV file
+    assert.deepEqual(
+      await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1]}),
+      ["$1.00", "1.20E3", "2,000,000", "43%", "(56)"],
+    );
+
+    const records = await api.getRecords('Formatted_numbers');
+    const cols = await api.getRecords('_grist_Tables_column');
+
+    // Actual data has correct values, e.g. 43% -> 0.43
+    assert.deepEqual(records, [{
+      id: 1,
+      fields: {
+        fn_currency: 1,
+        fn_scientific: 1200,
+        fn_decimal: 2000000,
+        fn_percent: 0.43,
+        fn_parens: -56,
+      },
+    }]);
+
+    // Get the fields we care about describing the columns to allow comparison.
+    // All column names in the CSV file start with "fn_"
+    const colFields = cols.map(
+      ({fields: {colId, type, widgetOptions}}) =>
+        ({colId, type, widgetOptions: JSON.parse(widgetOptions as string || "{}")})
+    ).filter(f => (f.colId as string).startsWith("fn_"));
+
+    // All the columns are numeric and have some kind of formatting
+    assert.deepEqual(colFields, [
+      {
+        colId: 'fn_currency',
+        type: 'Numeric',
+        widgetOptions: {decimals: 2, numMode: 'currency'}
+      },
+      {
+        colId: 'fn_scientific',
+        type: 'Numeric',
+        widgetOptions: {decimals: 2, numMode: 'scientific'}
+      },
+      {
+        colId: 'fn_decimal',
+        type: 'Numeric',
+        widgetOptions: {numMode: 'decimal'}
+      },
+      {
+        colId: 'fn_percent',
+        type: 'Numeric',
+        widgetOptions: {numMode: 'percent'}
+      },
+      {
+        colId: 'fn_parens',
+        type: 'Numeric',
+        widgetOptions: {numSign: 'parens'}
+      },
+    ]);
+
+    // Remove the imported table
+    await gu.undo();
+  });
+
+  it("should not show skip option for single table", async function() {
+    async function noSkip() {
+      await waitForDiffPreviewToLoad();
+      assert.isFalse(await driver.find('.test-importer-target-skip').isPresent());
+      await driver.sendKeys(Key.ESCAPE);
+      await driver.find('.test-modal-cancel').click();
+      await gu.waitAppFocus();
+    }
+    await gu.importFileDialog('./uploads/UploadedData1.csv');
+    await noSkip();
+    await gu.importFileDialog('./uploads/BooleanData.xlsx');
+    await noSkip();
+  });
+
+  it("should show skip option for multiple tables", async function() {
+    async function hasSkip() {
+      await waitForDiffPreviewToLoad();
+      assert.isTrue(await driver.find('.test-importer-target-skip').isDisplayed());
+      await driver.find('.test-importer-source-not-selected').click();
+      assert.isTrue(await driver.find('.test-importer-target-skip').isDisplayed());
+      await driver.find('.test-modal-cancel').click();
+      await gu.waitAppFocus();
+    }
+    await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv');
+    await hasSkip();
+    await gu.importFileDialog('./uploads/homicide_rates.xlsx');
+    await hasSkip();
+  });
+
+  it("should skip importing", async function() {
+    await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv');
+    await waitForDiffPreviewToLoad();
+    // Skip the first table.
+    await driver.find('.test-importer-target-skip').click();
+    // Make sure preview is grayed out.
+    assert.isTrue(await driver.find(".test-importer-preview-overlay").isPresent());
+    await driver.find('.test-modal-confirm').click();
+    await gu.waitAppFocus();
+    // Make sure only second table is visible.
+    assert.deepEqual(await gu.getPageNames(), ['Table1', 'UploadedData2']);
+    // And data is valid.
+    await gu.getPageItem('UploadedData2').click();
+    await gu.waitForServer();
+    assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5, 6]}),
+    [ 'BUS100',      'Intro to Business',   '',                    '01/13/2021',      '',
+      'BUS102',      'Business Law',        'Nathalie Patricia',   '01/13/2021',      '',
+      'BUS300',      'Business Operations', 'Michael Rian',        '01/14/2021',      '',
+      'BUS301',      'History of Business', 'Mariyam Melania',     '01/14/2021',      '',
+      'BUS500',      'Ethics and Law',      'Filip Andries',       '01/13/2021',      '',
+      'BUS540',      'Capstone',            '',                    '01/13/2021',      '' ]);
+    await gu.undo();
+  });
+
+  it("should clean mapping when skipped", async function() {
+    // Import UploadedData2 to have a destination table.
+    await gu.importFileDialog('./uploads/UploadedData2.csv');
+    await waitForDiffPreviewToLoad();
+    await driver.find('.test-modal-confirm').click();
+    await gu.waitAppFocus();
+
+    // Reimport
+    await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv');
+    await waitForDiffPreviewToLoad();
+
+    // Skip first table
+    await driver.find('.test-importer-target-skip').click();
+
+    // Select second table and add mapping to update existing records.
+    await driver.find('.test-importer-source-not-selected').click();
+    await driver.findContent('.test-importer-target-existing-table', /UploadedData2/).click();
+
+    await waitForDiffPreviewToLoad();
+    await waitForColumnMapping();
+    await driver.find('.test-importer-update-existing-records').click();
+    await driver.find('.test-importer-merge-fields-select').click();
+    await driver.findContent(
+      '.test-multi-select-menu .test-multi-select-menu-option',
+      /CourseId/
+    ).click();
+    await gu.sendKeys(Key.ESCAPE);
+    await gu.waitForServer();
+
+    // Now skip and make sure options are hidden
+    await openTableMapping();
+    await driver.find('.test-importer-target-skip').click();
+
+    // And unskip, and make sure options are back, but not filled
+    await driver.findContent('.test-importer-target-existing-table', /UploadedData2/).click();
+    await waitForDiffPreviewToLoad();
+
+    await waitForColumnMapping();
+    assert.isTrue(await driver.find('.test-importer-update-existing-records').isPresent());
+    assert.isTrue(await driver.find('.test-importer-merge-fields-select').isPresent());
+    assert.isTrue(await driver.find('.test-importer-merge-fields-message').isPresent());
+    assert.equal(await driver.find('.test-importer-merge-fields-select').getText(),
+      'Select fields to match on');
+
+    await driver.find('.test-modal-cancel').click();
+    await gu.waitAppFocus();
+    await gu.undo(2); // Press two times, as we cancelled and import hasn't cleaned temps.
+  });
+
+  it("should disable import button when all tables are skipped", async function() {
+    await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv');
+    await waitForDiffPreviewToLoad();
+    // Make sure both previews are available
+    for(const source of await driver.findAll(".test-importer-source")) {
+      await source.click();
+      assert.isFalse(await driver.find(".test-importer-preview-overlay").isPresent());
+    }
+    const sources = await driver.findAll(".test-importer-source");
+    // Skip both tables.
+     for(const source of sources) {
+      await source.click();
+      await gu.waitForServer();
+      await driver.find('.test-importer-target-skip').click();
+      await gu.waitForServer();
+    }
+    assert.equal(await driver.find('.test-modal-confirm').getAttribute('disabled'), 'true');
+    // Make sure both previews are grayed out
+    for(const source of sources) {
+      await source.click();
+      assert.isTrue(await driver.find(".test-importer-preview-overlay").isPresent());
+    }
+
+    // Enable first, and test if one is grayed out and the second is not.
+    await sources[0].click();
+    await gu.waitForServer();
+    await driver.find(".test-importer-target-new-table").click();
+    await gu.waitForServer();
+    await waitForDiffPreviewToLoad();
+    assert.isFalse(await driver.find(".test-importer-preview-overlay").isPresent());
+    assert.equal(await driver.find('.test-modal-confirm').getAttribute('disabled'), null);
+
+    // Second should be still grayed out
+    await sources[1].click();
+    assert.isTrue(await driver.find(".test-importer-preview-overlay").isPresent());
+
+    await driver.find('.test-modal-cancel').click();
+    await gu.waitAppFocus();
+  });
+
+  describe('when importing JSON', async function() {
+    // A previous bug caused an error to be thrown when finishing importing a nested JSON file.
+    it('should import successfully to new tables', async function() {
+      // Import a nested JSON file.
+      await gu.importFileDialog('./uploads/names.json');
+      assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true);
+
+      // Check that two preview tables were created.
+      assert.lengthOf(await driver.findAll('.test-importer-source'), 2);
+      assert.equal(
+        await driver.find('.test-importer-source[class*=-selected] .test-importer-from').getText(),
+        'names - names.json'
+      );
+      assert.deepEqual(
+        await driver.findAll('.test-importer-source .test-importer-from', (e) => e.getText()),
+        ['names - names.json', 'names_name - names.json']
+      );
+
+      // Check that the first table looks ok.
+      assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3]), [ '[1]', '[2]', '']);
+
+      // Check that the second table looks ok.
+      await driver.findContent('.test-importer-source', /names_name/).click();
+      await gu.waitForServer();
+      assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3]), [ 'Bob', 'Alice', '']);
+
+      // Finish import, and verify the import succeeded.
+      await driver.find('.test-modal-confirm').click();
+      await gu.waitForServer();
+      assert.deepEqual(await gu.getPageNames(), [
+        'Table1',
+        'names',
+        'names_name',
+      ]);
+
+      // Verify data was imported to Names correctly.
+      assert.deepEqual(
+        await gu.getVisibleGridCells({ rowNums: [1, 2, 3], cols: [0] }),
+        ['Names_name[1]', 'Names_name[2]', '']
+      );
+
+      // Open the side panel and check that the column type for 'name' is Reference (pointing to 'first').
+      await gu.toggleSidePanel('right', 'open');
+      await driver.find('.test-right-tab-field').click();
+      assert.equal(await driver.find('.test-fbuilder-type-select').getText(), 'Reference');
+      assert.equal(await gu.getRefTable(), 'names_name');
+      assert.equal(await gu.getRefShowColumn(), 'Row ID');
+
+      // Verify data was imported to Names_name correctly.
+      await gu.getPageItem('names_name').click();
+      await gu.waitForServer();
+      assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3], cols: [0] }), ['Bob', 'Alice', '']);
+    });
+
+    it('should import successfully to existing tables with references', async function() {
+      // Import the same nested JSON file again.
+      await gu.importFileDialog('./uploads/names.json');
+      assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true);
+
+      // Change the destination of both source tables to the existing destination ones.
+      await driver.findContent('.test-importer-target-existing-table', /Names/).click();
+      await gu.waitForServer();
+      // Now on the second tab.
+      await driver.findContent('.test-importer-source', /names_name/).click();
+      await driver.findContent('.test-importer-target-existing-table', /Names_name/).click();
+      await gu.waitForServer();
+
+      // Finish import, and verify the import succeeded.
+      await driver.find('.test-modal-confirm').click();
+      await gu.waitForServer();
+      assert.deepEqual(await gu.getPageNames(), [
+        'Table1',
+        'names',
+        'names_name',
+      ]);
+
+      // Verify data was imported to Names correctly.
+      assert.deepEqual(
+        await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5], cols: [0] }),
+        ['Names_name[1]', 'Names_name[2]', 'Names_name[1]', 'Names_name[2]', '']
+      );
+
+      // Open the side panel and check that the column type for 'name' is Reference (pointing to 'first').
+      await gu.toggleSidePanel('right', 'open');
+      await driver.find('.test-right-tab-field').click();
+      assert.equal(await driver.find('.test-fbuilder-type-select').getText(), 'Reference');
+      assert.equal(await gu.getRefTable(), 'names_name');
+      assert.equal(await gu.getRefShowColumn(), 'Row ID');
+
+      // Verify data was imported to Names_name correctly.
+      await gu.getPageItem('names_name').click();
+      await gu.waitForServer();
+      assert.deepEqual(await gu.getVisibleGridCells(
+        { rowNums: [1, 2, 3, 4, 5], cols: [0] }),
+        ['Bob', 'Alice', 'Bob', 'Alice', '']
+      );
+
+      // Undo the last 2 imports.
+      await gu.undo(2);
+    });
+  });
+
+  describe('when matching columns', async function() {
+    it('should not display column matching section for new destinations', async function() {
+      // Import an Excel file with multiple sheets.
+      await gu.importFileDialog('./uploads/World-v0.xlsx');
+      assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true);
+
+      // Check that the column matching section is not shown.
+      assert.isFalse(await driver.find('.test-importer-column-match-options').isPresent());
+    });
+
+    it('should display column matching section for existing destinations', async function() {
+      // From the previous test: finish importing World-v1.xlsx so we have tables to import to.
+      await driver.find('.test-modal-confirm').click();
+      await gu.waitForServer(10_000);
+
+      // Import the same file again.
+      await gu.importFileDialog('./uploads/World-v0.xlsx');
+      assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true);
+
+      // Change the destination of the selected sheet to the table created earlier.
+      await driver.findContent('.test-importer-target-existing-table', /Table1_2/).click();
+      await gu.waitForServer();
+
+      await waitForColumnMapping();
+      // Check that source and destination are populated for each column from the first sheet.
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { destination: 'a', source: 'a' },
+        { destination: 'b', source: 'b' },
+        { destination: 'c', source: 'c' },
+        { destination: 'd', source: 'd' },
+        { destination: 'E', source: 'E' },
+      ]);
+      assert.isFalse(await driver.find('.test-importer-unmatched-fields').isPresent());
+
+      // Switch to the City sheet, and check that the column matching section is no longer shown.
+      await driver.findContent('.test-importer-source', /City/).click();
+      await gu.waitForServer();
+      assert.isFalse(await driver.find('.test-importer-column-match-options').isPresent());
+
+      // Change the destination to 'City', and now check that the section is shown.
+      await driver.findContent('.test-importer-target-existing-table', /City/).click();
+      await gu.waitForServer();
+
+      await waitForColumnMapping();
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { destination: 'Name', source: 'Name' },
+        { destination: 'District', source: 'District' },
+        { destination: 'Population', source: 'Population' },
+        { destination: 'Country', source: 'Country' },
+        { destination: 'Pop. \'000', source: 'Pop. \'000' },
+      ]);
+      assert.isFalse(await driver.find('.test-importer-unmatched-fields').isPresent());
+    });
+
+    it('should allow skipping importing columns', async function() {
+      // Starting from the City sheet, open the menu for "Pop. '000".
+      await driver.findContent('.test-importer-column-match-source', /Pop\. '000/).click();
+
+      // Check that the menu contains only the selected source column, plus a 'Skip' option.
+      const menu = driver.find('.test-select-menu');
+      assert.deepEqual(
+        await menu.findAll('.test-importer-column-match-menu-item', el => el.getText()),
+        ['Skip', 'Pop. \'000']
+      );
+
+      // Click 'Skip', and check that the column mapping section and preview both updated.
+      await menu.findContent('.test-importer-column-match-menu-item', /Skip/).click();
+      await gu.waitForServer();
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { destination: 'Name', source: 'Name' },
+        { destination: 'District', source: 'District' },
+        { destination: 'Population', source: 'Population' },
+        { destination: 'Country', source: 'Country' },
+        { destination: 'Pop. \'000', source: 'Skip' },
+      ]);
+      assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [
+        'Kabul',  'Kabol',  '1780000',  '2',  '0',
+        'Qandahar',  'Qandahar',  '237500',  '2',  '0',
+        'Herat',  'Herat',  '186800',  '2',  '0',
+        'Mazar-e-Sharif',  'Balkh',  '127800',  '2',  '0',
+        'Amsterdam',  'Noord-Holland',  '731200',  '159',  '0',
+        'Rotterdam',  'Zuid-Holland',  '593321',  '159',  '0',
+      ]);
+
+      // Check that a message is now shown about there being 1 unmapped field.
+      assert.equal(
+        await driver.find('.test-importer-unmatched-fields').getText(),
+        '1 unmatched field in import:\nPop. \'000'
+      );
+
+      // Click Country in the column mapping section, and clear the formula.
+      await driver.findContent('.test-importer-column-match-source', /Country/).click();
+      await driver.find('.test-importer-apply-formula').click();
+      await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, Key.ENTER);
+      await gu.waitForServer();
+
+      // Check that the column mapping section and preview now show that it will be skipped.
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { destination: 'Name', source: 'Name' },
+        { destination: 'District', source: 'District' },
+        { destination: 'Population', source: 'Population' },
+        { destination: 'Country', source: 'Skip' },
+        { destination: 'Pop. \'000', source: 'Skip' },
+      ]);
+      assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [
+        'Kabul',  'Kabol',  '1780000',  '0',  '0',
+        'Qandahar',  'Qandahar',  '237500',  '0',  '0',
+        'Herat',  'Herat',  '186800',  '0',  '0',
+        'Mazar-e-Sharif',  'Balkh',  '127800',  '0',  '0',
+        'Amsterdam',  'Noord-Holland',  '731200',  '0',  '0',
+        'Rotterdam',  'Zuid-Holland',  '593321',  '0',  '0',
+      ]);
+      assert.equal(
+        await driver.find('.test-importer-unmatched-fields').getText(),
+        '2 unmatched fields in import:\nCountry, Pop. \'000'
+      );
+    });
+
+    it('should autocomplete formula in source', async function() {
+      // Starting from the City sheet, open the menu for "Pop. '000".
+      await openSourceFor(/Pop\. '000/);
+
+      // We want to map the same column twice, which is not possible through the menu, so we will
+      // use the formula.
+      await driver.find('.test-importer-apply-formula').click();
+      await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, '$Population', Key.ENTER);
+      await gu.waitForServer();
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { source: 'Name', destination: 'Name' },
+        { source: 'District', destination: 'District' },
+        { source: 'Population', destination: 'Population' },
+        { source: 'Skip', destination: 'Country' },
+        { source: '$Population\n', destination: 'Pop. \'000' },
+      ]);
+      assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [
+        'Kabul',  'Kabol',  '1780000',  '0',  '1780000',
+        'Qandahar',  'Qandahar',  '237500',  '0',  '237500',
+        'Herat',  'Herat',  '186800',  '0',  '186800',
+        'Mazar-e-Sharif',  'Balkh',  '127800',  '0',  '127800',
+        'Amsterdam',  'Noord-Holland',  '731200',  '0',  '731200',
+        'Rotterdam',  'Zuid-Holland',  '593321',  '0',  '593321',
+      ]);
+      assert.equal(
+        await driver.find('.test-importer-unmatched-fields').getText(),
+        '1 unmatched field in import:\nCountry'
+      );
+
+      // Click Country (with formula 'Skip') in the column mapping section, and start typing a formula.
+      await openSourceFor(/Country/);
+      await driver.find('.test-importer-apply-formula').click();
+      await gu.sendKeys('$');
+      await gu.waitForServer();
+
+      // Wait until the Ace autocomplete menu is shown.
+      await driver.wait(() => driver.find('div.ace_autocomplete').isDisplayed(), 2000);
+
+      // Check that the autocomplete is suggesting column ids from the imported table.
+      const completions = await driver.findAll(
+        'div.ace_autocomplete div.ace_line', async el => (await el.getText()).split(' ')[0]
+      );
+      assert.deepEqual(
+        completions.slice(0, 6),
+        [
+          "$\nCountry",
+          "$\nDistrict",
+          "$\nid",
+          "$\nName",
+          "$\nPop_000",
+          "$\nPopulation",
+        ]
+      );
+
+      // Set a constant value for the formula.
+      await gu.sendKeys(Key.BACK_SPACE, '123', Key.ENTER);
+      await gu.waitForServer();
+
+      // Check that the formula code is shown, as well as the evaluation result in the preview.
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { source: 'Name', destination: 'Name' },
+        { source: 'District', destination: 'District' },
+        { source: 'Population', destination: 'Population' },
+        { source: '123\n', destination: 'Country' },
+        { source: '$Population\n', destination: 'Pop. \'000' },
+      ]);
+
+      assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [
+        'Kabul',  'Kabol',  '1780000',  '123',  '1780000',
+        'Qandahar',  'Qandahar',  '237500',  '123',  '237500',
+        'Herat',  'Herat',  '186800',  '123',  '186800',
+        'Mazar-e-Sharif',  'Balkh',  '127800',  '123',  '127800',
+        'Amsterdam',  'Noord-Holland',  '731200',  '123',  '731200',
+        'Rotterdam',  'Zuid-Holland',  '593321',  '123',  '593321',
+      ]);
+      assert.isFalse(await driver.find('.test-importer-unmatched-fields').isPresent());
+    });
+
+    it('should reflect mappings when import to new table is finished', async function() {
+      // Skip 'Population', so that we can test imports with skipped columns.
+      await openSourceFor(/Population/);
+      await driver.findContent('.test-importer-column-match-menu-item', 'Skip').click();
+      await gu.waitForServer();
+
+      // Finish importing, and check that the destination tables have the correct data.
+      await driver.find('.test-modal-confirm').click();
+      await gu.waitForServer();
+      assert.deepEqual(await gu.getPageNames(), [
+        'Table1',
+        'Table1',
+        'City',
+        'Country',
+        'CountryLanguage',
+        'Country',
+        'CountryLanguage'
+      ]);
+
+      // Check the contents of Table1; it should have duplicates of the original 2 rows.
+      assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5], cols: [0, 1, 2, 3, 4] }),
+        [
+          'hello', '', '', '', 'HELLO',
+          '', 'world', '', '', '',
+          'hello', '', '', '', 'HELLO',
+          '', 'world', '', '', '',
+          '', '', '', '', '',
+        ]
+      );
+
+      await gu.getPageItem('City').click();
+      await gu.waitForServer();
+
+      // The first half should be the original imported rows.
+      assert.deepEqual(await gu.getVisibleGridCells(
+        {cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5, 6]}),
+        [
+          'Kabul', 'Kabol', '1780000', '2', '1780',
+          'Qandahar', 'Qandahar', '237500', '2', '237.5',
+          'Herat', 'Herat', '186800', '2', '186.8',
+          'Mazar-e-Sharif', 'Balkh', '127800', '2', '127.8',
+          'Amsterdam', 'Noord-Holland', '731200', '159', '731.2',
+          'Rotterdam', 'Zuid-Holland', '593321', '159', '593.321',
+        ]
+      );
+
+      // The second half should be the newly imported rows with custom mappings.
+      assert.equal(await gu.getGridRowCount(), 8159);
+      assert.deepEqual(await gu.getVisibleGridCells(
+        {cols: [0, 1, 2, 3, 4], rowNums: [8152, 8153, 8154, 8155, 8156, 8157]}),
+        [
+          'Gweru', 'Midlands', '0', '123', '128037',
+          'Gaza', 'Gaza', '0', '123', '353632',
+          'Khan Yunis', 'Khan Yunis', '0', '123', '123175',
+          'Hebron', 'Hebron', '0', '123', '119401',
+          'Jabaliya', 'North Gaza', '0', '123', '113901',
+          'Nablus', 'Nablus', '0', '123', '100231',
+        ]
+      );
+    });
+
+    it('should reflect mappings in previews of incremental imports', async function() {
+      // Delete the first row of the Country column. (Needed for a later assertion.)
+      await gu.sendKeys(Key.chord(await gu.modKey(), Key.UP));
+      await gu.getCell(3, 1).click();
+      await gu.sendKeys(Key.DELETE);
+      await gu.waitForServer();
+
+      // Import a CSV file containing city data, with column names that differ from the City table.
+      await gu.importFileDialog('./uploads/Cities.csv');
+      assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true);
+
+      // Change the destination to City, and check that column mapping defaults to skipping all columns.
+      await driver.findContent('.test-importer-target-existing-table', /City/).click();
+      await gu.waitForServer();
+      await waitForColumnMapping();
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { source: 'Skip', destination: 'Name' },
+        { source: 'Skip', destination: 'District' },
+        { source: 'Skip', destination: 'Population' },
+        { source: 'Skip', destination: 'Country' },
+        { source: 'Skip', destination: 'Pop. \'000' },
+      ]);
+
+      assert.equal(
+        await driver.find('.test-importer-unmatched-fields').getText(),
+        '5 unmatched fields in import:\nName, District, Population, Country, Pop. \'000'
+      );
+
+      // Set formula for 'Name' to 'city_name' by typing in the formula.
+      await openSourceFor(/Name/);
+      await driver.find('.test-importer-apply-formula').click();
+      await gu.sendKeys('$city_name', Key.ENTER);
+      await gu.waitForServer();
+
+      // Map 'District' to 'city_district' via the column mapping menu.
+      await openSourceFor('District');
+      const menu = driver.find('.test-select-menu');
+      await menu.findContent('.test-importer-column-match-menu-item', /city_district/).click();
+      await gu.waitForServer();
+
+      // Check the column mapping section and preview both updated correctly.
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { source: '$city_name\n', destination: 'Name' },
+        { source: 'city_district', destination: 'District' },
+        { source: 'Skip', destination: 'Population' },
+        { source: 'Skip', destination: 'Country' },
+        { source: 'Skip', destination: 'Pop. \'000' },
+      ]);
+      assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [
+        'Kabul',  'Kabol',  '0',  '0',  '0',
+        'Qandahar',  'Qandahar',  '0',  '0',  '0',
+        'Herat',  'Herat',  '0',  '0',  '0',
+        'Mazar-e-Sharif',  'Balkh',  '0',  '0',  '0',
+        'Amsterdam',  'Noord-Holland',  '0',  '0',  '0',
+        'Rotterdam',  'Zuid-Holland',  '0',  '0',  '0',
+      ]);
+      assert.equal(
+        await driver.find('.test-importer-unmatched-fields').getText(),
+        '3 unmatched fields in import:\nPopulation, Country, Pop. \'000'
+      );
+
+      // Now toggle 'Update existing records', and merge on 'Name' and 'District'.
+      await driver.find('.test-importer-update-existing-records').click();
+      await driver.find('.test-importer-merge-fields-select').click();
+      await driver.findContent(
+        '.test-multi-select-menu .test-multi-select-menu-option',
+        /Name/
+      ).click();
+      await driver.findContent(
+        '.test-multi-select-menu .test-multi-select-menu-option',
+        /District/
+      ).click();
+      await gu.sendKeys(Key.ESCAPE);
+      await gu.waitForServer();
+
+      // Check that the column mapping section and preview updated correctly.
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { source: '$city_name\n', destination: 'Name' },
+        { source: 'city_district', destination: 'District' },
+        { source: 'Skip', destination: 'Population' },
+        { source: 'Skip', destination: 'Country' },
+        { source: 'Skip', destination: 'Pop. \'000' },
+      ]);
+      await waitForDiffPreviewToLoad();
+      assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5]), [
+        'Kabul', 'Kabol', [undefined, undefined, '1780000'], '', [undefined, undefined, '1780'],
+        'Qandahar', 'Qandahar', [undefined, undefined, '237500'], [undefined, undefined, '2'],
+          [undefined, undefined, '237.5'],
+        'Herat', 'Herat', [undefined, undefined, '186800'], [undefined, undefined, '2'],
+          [undefined, undefined, '186.8'],
+        'Mazar-e-Sharif', 'Balkh', [undefined, undefined, '127800'], [undefined, undefined, '2'],
+          [undefined, undefined, '127.8'],
+        'Amsterdam', 'Noord-Holland', [undefined, undefined, '731200'], [undefined, undefined, '159'],
+          [undefined, undefined, '731.2'],
+      ]);
+      assert.equal(
+        await driver.find('.test-importer-unmatched-fields').getText(),
+        '3 unmatched fields in import:\nPopulation, Country, Pop. \'000'
+      );
+
+      // Map the remaining columns, except "Country"; we'll leave it skipped to check that
+      // we don't overwrite any values in the destination table. (A previous bug caused non-text
+      // skipped columns to overwrite data with default values, like 0.)
+      await openSourceFor(/Population/);
+      await driver.find('.test-importer-apply-formula').click();
+      await gu.sendKeys('$city_pop', Key.ENTER);
+      await gu.waitForServer();
+
+      // For "Pop. '000", deliberately map a duplicate column (so we can later check if import succeeded).
+      await openSourceFor(/Pop\. '000/);
+      await driver.find('.test-importer-apply-formula').click();
+      await gu.sendKeys('$city_pop', Key.ENTER);
+      await gu.waitForServer();
+
+      assert.deepEqual(await getColumnMatchingRows(), [
+        { source: '$city_name\n',      destination: 'Name' },
+        { source: 'city_district',   destination: 'District' },
+        { source: '$city_pop\n',       destination: 'Population' },
+        { source: 'Skip',            destination: 'Country' },
+        { source: '$city_pop\n',       destination: 'Pop. \'000' },
+      ]);
+
+      await waitForDiffPreviewToLoad();
+      assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5]), [
+        // Kabul's Country column should appear blank, since we deleted it earlier.
+        'Kabul', 'Kabol', ['1780000', '3560000', undefined], '', ['1780', '3560000', undefined],
+        'Qandahar', 'Qandahar', ['237500', '475000', undefined], [undefined, undefined, '2'],
+          ['237.5', '475000', undefined],
+        'Herat', 'Herat', ['186800', '373600', undefined], [undefined, undefined, '2'],
+          ['186.8', '373600', undefined],
+        'Mazar-e-Sharif', 'Balkh', ['127800', '255600', undefined], [undefined, undefined, '2'],
+          ['127.8', '255600', undefined],
+        'Amsterdam', 'Noord-Holland', ['731200', '1462400', undefined], [undefined, undefined, '159'],
+          ['731.2', '1462400', undefined],
+      ]);
+      assert.equal(
+        await driver.find('.test-importer-unmatched-fields').getText(),
+        '1 unmatched field in import:\nCountry'
+      );
+    });
+
+    it('should reflect mappings when incremental import is finished', async function() {
+      // Finish importing, and check that the destination table has the correct data.
+      await driver.find('.test-modal-confirm').click();
+      await gu.waitForServer();
+      assert.deepEqual(await gu.getPageNames(), [
+        'Table1',
+        'Table1',
+        'City',
+        'Country',
+        'CountryLanguage',
+        'Country',
+        'CountryLanguage',
+      ]);
+
+      assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5]}),
+        [
+          // Kabul's Country column should still be blank, since we skipped it earlier.
+          'Kabul', 'Kabol', '3560000', '', '3560000',
+          'Qandahar', 'Qandahar', '475000', '2', '475000',
+          'Herat', 'Herat', '373600', '2', '373600',
+          'Mazar-e-Sharif', 'Balkh', '255600', '2', '255600',
+          'Amsterdam', 'Noord-Holland', '1462400', '159', '1462400',
+        ]
+      );
+    });
+  });
+});

test/nbrowser/importerTestUtils.ts

@@ -41,7 +41,7 @@ export const waitForDiffPreviewToLoad = stackWrapFunc(async (): Promise<void> =>
 // Helper that gets the list of visible column matching rows to the left of the preview.
 export const getColumnMatchingRows = stackWrapFunc(async (): Promise<{source: string, destination: string}[]> => {
   return await driver.findAll('.test-importer-column-match-source-destination', async (el) => {
-    const source = await el.find('.test-importer-column-match-formula').getText();
+    const source = await el.find('.test-importer-column-match-formula').getAttribute('textContent');
     const destination = await el.find('.test-importer-column-match-destination').getText();
     return {source, destination};
   });

test/server/lib/ACLFormula.ts

@@ -1,7 +1,7 @@
 import {CellValue} from 'app/common/DocActions';
-import {AclMatchFunc, InfoView} from 'app/common/GranularAccessClause';
+import {InfoView} from 'app/common/GranularAccessClause';
 import {GristObjCode} from 'app/plugin/GristData';
-import {compileAclFormula} from 'app/server/lib/ACLFormula';
+import {CompiledPredicateFormula, compilePredicateFormula} from 'app/common/PredicateFormula';
 import {makeExceptionalDocSession} from 'app/server/lib/DocSession';
 import {User} from 'app/server/lib/GranularAccess';
 import {assert} from 'chai';
@@ -26,7 +26,7 @@ describe('ACLFormula', function() {
 
   const V = getInfoView;    // A shortcut.
 
-  type SetAndCompile = (aclFormula: string) => Promise<AclMatchFunc>;
+  type SetAndCompile = (aclFormula: string) => Promise<CompiledPredicateFormula>;
   let setAndCompile: SetAndCompile;
 
   before(async function () {
@@ -44,7 +44,7 @@ describe('ACLFormula', function() {
         fakeSession, {tableId: '_grist_ACLRules', filters: {id: [ruleRef]}});
       assert(tableData[3].aclFormulaParsed, "Expected aclFormulaParsed to be populated");
       const parsedFormula = String(tableData[3].aclFormulaParsed[0]);
-      return compileAclFormula(JSON.parse(parsedFormula));
+      return compilePredicateFormula(JSON.parse(parsedFormula));
     };
   });