(core) Add dropdown conditions

Summary:
Dropdown conditions let you specify a predicate formula that's used to filter
choices and references in their respective autocomplete dropdown menus.

Test Plan: Python and browser tests (WIP).

Reviewers: jarek, paulfitz

Reviewed By: jarek

Subscribers: dsagal, paulfitz

Differential Revision: https://phab.getgrist.com/D4235

app/common/ACLRuleCollection.ts

@@ -3,14 +3,15 @@ import {AVAILABLE_BITS_COLUMNS, AVAILABLE_BITS_TABLES, trimPermissions} from 'ap
 import {ACLRulesReader} from 'app/common/ACLRulesReader';
 import {AclRuleProblem} from 'app/common/ActiveDocAPI';
 import {DocData} from 'app/common/DocData';
-import {AclMatchFunc, ParsedAclFormula, RulePart, RuleSet, UserAttributeRule} from 'app/common/GranularAccessClause';
+import {RulePart, RuleSet, UserAttributeRule} from 'app/common/GranularAccessClause';
 import {getSetMapValue, isNonNullish} from 'app/common/gutil';
+import {CompiledPredicateFormula, ParsedPredicateFormula} from 'app/common/PredicateFormula';
 import {MetaRowRecord} from 'app/common/TableData';
 import {decodeObject} from 'app/plugin/objtypes';
 
 export type ILogger = Pick<Console, 'log'|'debug'|'info'|'warn'|'error'>;
 
-const defaultMatchFunc: AclMatchFunc = () => true;
+const defaultMatchFunc: CompiledPredicateFormula = () => true;
 
 export const SPECIAL_RULES_TABLE_ID = '*SPECIAL';
 
@@ -20,12 +21,12 @@ const DEFAULT_RULE_SET: RuleSet = {
   colIds: '*',
   body: [{
     aclFormula: "user.Access in [EDITOR, OWNER]",
-    matchFunc:  (input) => ['editors', 'owners'].includes(String(input.user.Access)),
+    matchFunc: (input) => ['editors', 'owners'].includes(String(input.user!.Access)),
     permissions: parsePermissions('all'),
     permissionsText: 'all',
   }, {
     aclFormula: "user.Access in [VIEWER]",
-    matchFunc:  (input) => ['viewers'].includes(String(input.user.Access)),
+    matchFunc: (input) => ['viewers'].includes(String(input.user!.Access)),
     permissions: parsePermissions('+R-CUDS'),
     permissionsText: '+R',
   }, {
@@ -48,7 +49,7 @@ const SPECIAL_RULE_SETS: Record<string, RuleSet> = {
     colIds: ['SchemaEdit'],
     body: [{
       aclFormula: "user.Access in [EDITOR, OWNER]",
-      matchFunc:  (input) => ['editors', 'owners'].includes(String(input.user.Access)),
+      matchFunc: (input) => ['editors', 'owners'].includes(String(input.user!.Access)),
       permissions: parsePermissions('+S'),
       permissionsText: '+S',
     }, {
@@ -63,7 +64,7 @@ const SPECIAL_RULE_SETS: Record<string, RuleSet> = {
     colIds: ['AccessRules'],
     body: [{
       aclFormula: "user.Access in [OWNER]",
-      matchFunc:  (input) => ['owners'].includes(String(input.user.Access)),
+      matchFunc: (input) => ['owners'].includes(String(input.user!.Access)),
       permissions: parsePermissions('+R'),
       permissionsText: '+R',
     }, {
@@ -78,7 +79,7 @@ const SPECIAL_RULE_SETS: Record<string, RuleSet> = {
     colIds: ['FullCopies'],
     body: [{
       aclFormula: "user.Access in [OWNER]",
-      matchFunc:  (input) => ['owners'].includes(String(input.user.Access)),
+      matchFunc: (input) => ['owners'].includes(String(input.user!.Access)),
       permissions: parsePermissions('+R'),
       permissionsText: '+R',
     }, {
@@ -102,7 +103,7 @@ const EMERGENCY_RULE_SET: RuleSet = {
   colIds: '*',
   body: [{
     aclFormula: "user.Access in [OWNER]",
-    matchFunc:  (input) => ['owners'].includes(String(input.user.Access)),
+    matchFunc:  (input) => ['owners'].includes(String(input.user!.Access)),
     permissions: parsePermissions('all'),
     permissionsText: 'all',
   }, {
@@ -381,7 +382,7 @@ export class ACLRuleCollection {
 
 export interface ReadAclOptions {
   log: ILogger;     // For logging warnings during rule processing.
-  compile?: (parsed: ParsedAclFormula) => AclMatchFunc;
+  compile?: (parsed: ParsedPredicateFormula) => CompiledPredicateFormula;
   // If true, add and modify access rules in some special ways.
   // Specifically, call addHelperCols to add helper columns of restricted columns to rule sets,
   // and use ACLShareRules to implement any special shares as access rules.

app/common/ACLRulesReader.ts

@@ -3,7 +3,8 @@ import { getSetMapValue } from 'app/common/gutil';
 import { SchemaTypes } from 'app/common/schema';
 import { ShareOptions } from 'app/common/ShareOptions';
 import { MetaRowRecord, MetaTableData } from 'app/common/TableData';
-import { isEqual, sortBy } from 'lodash';
+import isEqual from 'lodash/isEqual';
+import sortBy from 'lodash/sortBy';
 
 /**
  * For special shares, we need to refer to resources that may not

app/common/ActiveDocAPI.ts

@@ -1,6 +1,6 @@
 import {ActionGroup} from 'app/common/ActionGroup';
 import {BulkAddRecord, CellValue, TableDataAction, UserAction} from 'app/common/DocActions';
-import {FormulaProperties} from 'app/common/GranularAccessClause';
+import {PredicateFormulaProperties} from 'app/common/PredicateFormula';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {DocStateComparison, PermissionData, UserAccessData} from 'app/common/UserAPI';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
@@ -421,7 +421,7 @@ export interface ActiveDocAPI {
    * Find and return a list of auto-complete suggestions that start with `txt`, when editing a
    * formula in table `tableId` and column `columnId`.
    */
-  autocomplete(txt: string, tableId: string, columnId: string, rowId: UIRowId): Promise<ISuggestionWithValue[]>;
+  autocomplete(txt: string, tableId: string, columnId: string, rowId: UIRowId | null): Promise<ISuggestionWithValue[]>;
 
   /**
    * Removes the current instance from the doc.
@@ -467,7 +467,7 @@ export interface ActiveDocAPI {
   /**
    * Check if an ACL formula is valid. If not, will throw an error with an explanation.
    */
-  checkAclFormula(text: string): Promise<FormulaProperties>;
+  checkAclFormula(text: string): Promise<PredicateFormulaProperties>;
 
   /**
    * Get a token for out-of-band access to the document.

app/common/ColumnFilterFunc.ts

@@ -4,7 +4,7 @@ import {decodeObject} from "app/plugin/objtypes";
 import moment, { Moment } from "moment-timezone";
 import {extractInfoFromColType, isDateLikeType, isList, isListType, isNumberType} from "app/common/gristTypes";
 import {isRelativeBound, relativeDateToUnixTimestamp} from "app/common/RelativeDates";
-import {noop} from "lodash";
+import noop from "lodash/noop";
 
 export type ColumnFilterFunc = (value: CellValue) => boolean;
 

app/common/DropdownCondition.ts

@@ -0,0 +1,20 @@
+import { CompiledPredicateFormula } from 'app/common/PredicateFormula';
+
+export interface DropdownCondition {
+  text: string;
+  parsed: string;
+}
+
+export type DropdownConditionCompilationResult =
+  | DropdownConditionCompilationSuccess
+  | DropdownConditionCompilationFailure;
+
+interface DropdownConditionCompilationSuccess {
+  kind: 'success';
+  result: CompiledPredicateFormula;
+}
+
+interface DropdownConditionCompilationFailure {
+  kind: 'failure';
+  error: string;
+}

app/common/GranularAccessClause.ts

@@ -1,7 +1,8 @@
 import {PartialPermissionSet} from 'app/common/ACLPermissions';
 import {CellValue, RowRecord} from 'app/common/DocActions';
+import {CompiledPredicateFormula} from 'app/common/PredicateFormula';
+import {Role} from 'app/common/roles';
 import {MetaRowRecord} from 'app/common/TableData';
-import {Role} from './roles';
 
 export interface RuleSet {
   tableId: '*' | string;
@@ -18,7 +19,7 @@ export interface RulePart {
   permissionsText: string;        // The text version of PermissionSet, as stored.
 
   // Compiled version of aclFormula.
-  matchFunc?: AclMatchFunc;
+  matchFunc?: CompiledPredicateFormula;
 
   // Optional memo, currently extracted from comment in formula.
   memo?: string;
@@ -53,35 +54,6 @@ export interface UserInfo {
   toJSON(): {[key: string]: any};
 }
 
-/**
- * Input into the AclMatchFunc. Compiled formulas evaluate AclMatchInput to produce a boolean.
- */
-export interface AclMatchInput {
-  user: UserInfo;
-  rec?: InfoView;
-  newRec?: InfoView;
-  docId?: string;
-}
-
-/**
- * The actual boolean function that can evaluate a request. The result of compiling ParsedAclFormula.
- */
-export type AclMatchFunc = (input: AclMatchInput) => boolean;
-
-/**
- * Representation of a parsed ACL formula.
- */
-type PrimitiveCellValue = number|string|boolean|null;
-export type ParsedAclFormula = [string, ...(ParsedAclFormula|PrimitiveCellValue)[]];
-
-/**
- * Observations about a formula.
- */
-export interface FormulaProperties {
-  hasRecOrNewRec?: boolean;
-  usedColIds?: string[];
-}
-
 export interface UserAttributeRule {
   origRecord?: RowRecord;         // Original record used to create this UserAttributeRule.
   name: string;       // Should be unique among UserAttributeRules.
@@ -89,45 +61,3 @@ export interface UserAttributeRule {
   lookupColId: string;  // Column in tableId in which to do the lookup.
   charId: string;     // Attribute to look up, possibly a path. E.g. 'Email' or 'office.city'.
 }
-
-/**
- * Check some key facts about the formula.
- */
-export function getFormulaProperties(formula: ParsedAclFormula) {
-  const result: FormulaProperties = {};
-  if (usesRec(formula)) { result.hasRecOrNewRec = true; }
-  const colIds = new Set<string>();
-  collectRecColIds(formula, colIds);
-  result.usedColIds = Array.from(colIds);
-  return result;
-}
-
-/**
- * Check whether a formula mentions `rec` or `newRec`.
- */
-export function usesRec(formula: ParsedAclFormula): boolean {
-  if (!Array.isArray(formula)) { throw new Error('expected a list'); }
-  if (isRecOrNewRec(formula)) {
-    return true;
-  }
-  return formula.some(el => {
-    if (!Array.isArray(el)) { return false; }
-    return usesRec(el);
-  });
-}
-
-function isRecOrNewRec(formula: ParsedAclFormula|PrimitiveCellValue): boolean {
-  return Array.isArray(formula) &&
-    formula[0] === 'Name' &&
-    (formula[1] === 'rec' || formula[1] === 'newRec');
-}
-
-function collectRecColIds(formula: ParsedAclFormula, colIds: Set<string>): void {
-  if (!Array.isArray(formula)) { throw new Error('expected a list'); }
-  if (formula[0] === 'Attr' && isRecOrNewRec(formula[1])) {
-    const colId = formula[2];
-    colIds.add(String(colId));
-    return;
-  }
-  formula.forEach(el => Array.isArray(el) && collectRecColIds(el, colIds));
-}

app/common/PredicateFormula.ts

@@ -0,0 +1,223 @@
+/**
+ * Representation and compilation of predicate formulas.
+ *
+ * An example of a predicate formula is: "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']".
+ * These formulas are parsed in Python into a tree with nodes of the form [NODE_TYPE, ...args].
+ * See sandbox/grist/predicate_formula.py for details.
+ *
+ * This module includes typings for the nodes, and the compilePredicateFormula() function that
+ * turns such trees into actual predicate functions.
+ */
+import {CellValue, RowRecord} from 'app/common/DocActions';
+import {ErrorWithCode} from 'app/common/ErrorWithCode';
+import {InfoView, UserInfo} from 'app/common/GranularAccessClause';
+import {decodeObject} from 'app/plugin/objtypes';
+import constant = require('lodash/constant');
+
+/**
+ * Representation of a parsed predicate formula.
+ */
+export type PrimitiveCellValue = number|string|boolean|null;
+export type ParsedPredicateFormula = [string, ...(ParsedPredicateFormula|PrimitiveCellValue)[]];
+
+/**
+ * Inputs to a predicate formula function.
+ */
+export interface PredicateFormulaInput {
+  user?: UserInfo;
+  rec?: RowRecord|InfoView;
+  newRec?: InfoView;
+  docId?: string;
+  choice?: string|RowRecord|InfoView;
+}
+
+export class EmptyRecordView implements InfoView {
+  public get(_colId: string): CellValue { return null; }
+  public toJSON() { return {}; }
+}
+
+/**
+ * The result of compiling ParsedPredicateFormula.
+ */
+export type CompiledPredicateFormula = (input: PredicateFormulaInput) => boolean;
+
+const GRIST_CONSTANTS: Record<string, string> = {
+  EDITOR: 'editors',
+  OWNER: 'owners',
+  VIEWER: 'viewers',
+};
+
+/**
+ * An intermediate predicate formula returned during compilation, which may return
+ * a non-boolean value.
+ */
+type IntermediatePredicateFormula = (input: PredicateFormulaInput) => any;
+
+export interface CompilePredicateFormulaOptions {
+  /** Defaults to `'acl'`. */
+  variant?: 'acl'|'dropdown-condition';
+}
+
+/**
+ * Compiles a parsed predicate formula and returns it.
+ */
+export function compilePredicateFormula(
+  parsedPredicateFormula: ParsedPredicateFormula,
+  options: CompilePredicateFormulaOptions = {}
+): CompiledPredicateFormula {
+  const {variant = 'acl'} = options;
+
+  function compileNode(node: ParsedPredicateFormula): IntermediatePredicateFormula {
+    const rawArgs = node.slice(1);
+    const args = rawArgs as ParsedPredicateFormula[];
+    switch (node[0]) {
+      case 'And':   { const parts = args.map(compileNode); return (input) => parts.every(p => p(input)); }
+      case 'Or':    { const parts = args.map(compileNode); return (input) => parts.some(p => p(input)); }
+      case 'Add':   return compileAndCombine(args, ([a, b]) => a + b);
+      case 'Sub':   return compileAndCombine(args, ([a, b]) => a - b);
+      case 'Mult':  return compileAndCombine(args, ([a, b]) => a * b);
+      case 'Div':   return compileAndCombine(args, ([a, b]) => a / b);
+      case 'Mod':   return compileAndCombine(args, ([a, b]) => a % b);
+      case 'Not':   return compileAndCombine(args, ([a]) => !a);
+      case 'Eq':    return compileAndCombine(args, ([a, b]) => a === b);
+      case 'NotEq': return compileAndCombine(args, ([a, b]) => a !== b);
+      case 'Lt':    return compileAndCombine(args, ([a, b]) => a < b);
+      case 'LtE':   return compileAndCombine(args, ([a, b]) => a <= b);
+      case 'Gt':    return compileAndCombine(args, ([a, b]) => a > b);
+      case 'GtE':   return compileAndCombine(args, ([a, b]) => a >= b);
+      case 'Is':    return compileAndCombine(args, ([a, b]) => a === b);
+      case 'IsNot': return compileAndCombine(args, ([a, b]) => a !== b);
+      case 'In':    return compileAndCombine(args, ([a, b]) => Boolean(b?.includes(a)));
+      case 'NotIn': return compileAndCombine(args, ([a, b]) => !b?.includes(a));
+      case 'List':  return compileAndCombine(args, (values) => values);
+      case 'Const': return constant(node[1] as CellValue);
+      case 'Name': {
+        const name = rawArgs[0] as keyof PredicateFormulaInput;
+        if (GRIST_CONSTANTS[name]) { return constant(GRIST_CONSTANTS[name]); }
+
+        let validNames: string[];
+        switch (variant) {
+          case 'acl': {
+            validNames = ['newRec', 'rec', 'user'];
+            break;
+          }
+          case 'dropdown-condition': {
+            validNames = ['rec', 'choice'];
+            break;
+          }
+        }
+        if (!validNames.includes(name)) { throw new Error(`Unknown variable '${name}'`); }
+
+        return (input) => input[name];
+      }
+      case 'Attr': {
+        const attrName = rawArgs[1] as string;
+        return compileAndCombine([args[0]], ([value]) => getAttr(value, attrName, args[0]));
+      }
+      case 'Comment': return compileNode(args[0]);
+    }
+    throw new Error(`Unknown node type '${node[0]}'`);
+  }
+
+  /**
+   * Helper for operators: compile a list of nodes, then when evaluating, evaluate them all and
+   * combine the array of results using the given combine() function.
+   */
+  function compileAndCombine(
+    args: ParsedPredicateFormula[],
+    combine: (values: any[]) => any
+  ): IntermediatePredicateFormula {
+    const compiled = args.map(compileNode);
+    return (input: PredicateFormulaInput) => combine(compiled.map(c => c(input)));
+  }
+
+  const compiledPredicateFormula = compileNode(parsedPredicateFormula);
+  return (input) => Boolean(compiledPredicateFormula(input));
+}
+
+function describeNode(node: ParsedPredicateFormula): string {
+  if (node[0] === 'Name') {
+    return node[1] as string;
+  } else if (node[0] === 'Attr') {
+    return describeNode(node[1] as ParsedPredicateFormula) + '.' + (node[2] as string);
+  } else {
+    return 'value';
+  }
+}
+
+function getAttr(value: any, attrName: string, valueNode: ParsedPredicateFormula): any {
+  if (value == null) {
+    if (valueNode[0] === 'Name' && (valueNode[1] === 'rec' || valueNode[1] === 'newRec')) {
+      // This code is recognized by GranularAccess to know when an ACL rule is row-specific.
+      throw new ErrorWithCode('NEED_ROW_DATA', `Missing row data '${valueNode[1]}'`);
+    }
+    throw new Error(`No value for '${describeNode(valueNode)}'`);
+  }
+  return typeof value.get === 'function'
+    ? decodeObject(value.get(attrName)) // InfoView
+    : value[attrName];
+}
+
+/**
+ * Predicate formula properties.
+ */
+export interface PredicateFormulaProperties {
+  /**
+   * List of column ids that are referenced by either `$` or `rec.` notation.
+   */
+  recColIds?: string[];
+  /**
+   * List of column ids that are referenced by `choice.` notation.
+   *
+   * Only applies to the `dropdown-condition` variant of predicate formulas,
+   * and only for Reference and Reference List columns.
+   */
+  choiceColIds?: string[];
+}
+
+/**
+ * Returns properties about a predicate `formula`.
+ *
+ * Properties include the list of column ids referenced in the formula.
+ * Currently, this information is used for error validation; specifically, to
+ * report when invalid column ids are referenced in ACL formulas and dropdown
+ * conditions.
+ */
+export function getPredicateFormulaProperties(
+  formula: ParsedPredicateFormula
+): PredicateFormulaProperties {
+  return {
+    recColIds: [...getRecColIds(formula)],
+    choiceColIds: [...getChoiceColIds(formula)],
+  };
+}
+
+function isRecOrNewRec(formula: ParsedPredicateFormula|PrimitiveCellValue): boolean {
+  return Array.isArray(formula) &&
+    formula[0] === 'Name' &&
+    (formula[1] === 'rec' || formula[1] === 'newRec');
+}
+
+function getRecColIds(formula: ParsedPredicateFormula): string[] {
+  return [...new Set(collectColIds(formula, isRecOrNewRec))];
+}
+
+function isChoice(formula: ParsedPredicateFormula|PrimitiveCellValue): boolean {
+  return Array.isArray(formula) && formula[0] === 'Name' && formula[1] === 'choice';
+}
+
+function getChoiceColIds(formula: ParsedPredicateFormula): string[] {
+  return [...new Set(collectColIds(formula, isChoice))];
+}
+
+function collectColIds(
+  formula: ParsedPredicateFormula,
+  isIdentifierWithColIds: (formula: ParsedPredicateFormula|PrimitiveCellValue) => boolean,
+): string[] {
+  if (!Array.isArray(formula)) { throw new Error('expected a list'); }
+  if (formula[0] === 'Attr' && isIdentifierWithColIds(formula[1])) {
+    const colId = String(formula[2]);
+    return [colId];
+  }
+  return formula.flatMap(el => Array.isArray(el) ? collectColIds(el, isIdentifierWithColIds) : []);
+}

app/common/RelativeDates.ts

@@ -2,9 +2,12 @@
 // time defined as a series of periods. Hence, starting from the current date, each one of the
 // periods gets applied successively which eventually yields to the final date. Typical relative
 
-import { isEqual, isNumber, isUndefined, omitBy } from "lodash";
-import moment from "moment-timezone";
 import getCurrentTime from "app/common/getCurrentTime";
+import isEqual from "lodash/isEqual";
+import isNumber from "lodash/isNumber";
+import isUndefined from "lodash/isUndefined";
+import omitBy from "lodash/omitBy";
+import moment from "moment-timezone";
 
 // Relative date uses one or two periods. When relative dates are defined by two periods, they are
 // applied successively to the start date to resolve the target date. In practice in grist, as of