(core) Show example values in formula autocomplete

Summary:
This diff adds a preview of the value of certain autocomplete suggestions, especially of the form `$foo.bar` or `user.email`. The main initial motivation was to show the difference between `$Ref` and `$Ref.DisplayCol`, but the feature is more general.

The client now sends the row ID of the row being edited (along with the table and column IDs which were already sent) to the server to fetch autocomplete suggestions. The returned suggestions are now tuples `(suggestion, example_value)` where `example_value` is a string or null. The example value is simply obtained by evaluating (in a controlled way) the suggestion in the context of the given record and the current user. The string representation is similar to the standard `repr` but dates and datetimes are formatted, and the whole thing is truncated for efficiency.

The example values are shown in the autocomplete popup separated from the actual suggestion by a number of spaces calculated to:

1. Clearly separate the suggestion from the values
2. Left-align the example values in most cases
3. Avoid having so much space such that connecting suggestions and values becomes visually difficult.

The tokenization of the row is then tweaked to show the example in light grey to deemphasise it.

Main discussion where the above was decided: https://grist.slack.com/archives/CDHABLZJT/p1661795588100009

The diff also includes various other small improvements and fixes:

- The autocomplete popup is much wider to make room for long suggestions, particularly lookups, as pointed out in https://phab.getgrist.com/D3580#inline-41007. The wide popup is the reason a fancy solution was needed to position the example values. I didn't see a way to dynamically resize the popup based on suggestions, and it didn't seem like a good idea to try.
- The `grist` and `python` labels previously shown on the right are removed. They were not helpful (https://grist.slack.com/archives/CDHABLZJT/p1659697086155179) and would get in the way of the example values.
- Fixed a bug in our custom tokenization that caused function arguments to be weirdly truncated in the middle: https://grist.slack.com/archives/CDHABLZJT/p1661956353699169?thread_ts=1661953258.342739&cid=CDHABLZJT and https://grist.slack.com/archives/C069RUP71/p1659696778991339
- Hide suggestions involving helper columns like `$gristHelper_Display` or `Table.lookupRecords(gristHelper_Display=` (https://grist.slack.com/archives/CDHABLZJT/p1661953258342739). The former has been around for a while and seems to be a mistake. The fix is simply to use `is_visible_column` instead of `is_user_column`. Since the latter is not used anywhere else, and using it in the first place seems like a mistake more than anything else, I've also removed the function to prevent similar mistakes in the future.
- Don't suggest private columns as lookup arguments: https://grist.slack.com/archives/CDHABLZJT/p1662133416652499?thread_ts=1661795588.100009&cid=CDHABLZJT
- Only fetch fresh suggestions specifically after typing `lookupRecords(` or `lookupOne(` rather than just `(`, as this would needlessly hide function suggestions which could still be useful to see the arguments. However this only makes a difference when there are still multiple matching suggestions, otherwise Ace hides them anyway.

Test Plan: Extended and updated several Python and browser tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3611

app/client/aclui/ACLFormulaEditor.ts

@@ -41,7 +41,7 @@ export function aclFormulaEditor(options: ACLFormulaOptions) {
   );
   editor.on("change", () => showPlaceholder.set(!editor.getValue().length));
 
-  async function getSuggestions(prefix: string) {
+  async function getSuggestions(prefix: string): Promise<Array<[string, null]>> {
     return [
       // The few Python keywords and constants we support.
       'and', 'or', 'not', 'in', 'is', 'True', 'False', 'None',
@@ -51,7 +51,7 @@ export function aclFormulaEditor(options: ACLFormulaOptions) {
       'user', 'rec', 'newRec',
       // Other completions that depend on doc schema or other rules.
       ...options.getSuggestions(prefix),
-    ];
+    ].map(suggestion => [suggestion, null]);  // null means no example value
   }
   setupAceEditorCompletions(editor, {getSuggestions});
 

app/client/components/AceEditor.css

@@ -8,6 +8,10 @@
   cursor: pointer;
 }
 
+.ace_grist_example {
+  color: #8f8f8f;
+}
+
 .ace_editor.ace_autocomplete .ace_completion-highlight.ace_grist_link {
   color: var(--grist-color-dark-green);
 }
@@ -16,3 +20,8 @@
   z-index: 7;
   pointer-events: auto;
 }
+
+.ace_editor.ace_autocomplete {
+  width: 500px !important;  /* the default in language_tools.js is 280px */
+  max-width: 80%;  /* of the screen, for hypothetical mobile support */
+}

app/client/components/AceEditor.js

@@ -189,9 +189,11 @@ AceEditor.prototype._setup = function() {
   this.editor = this.autoDisposeWith('destroy', ace.edit(this.editorDom));
   if (this.gristDoc && this.column) {
     const getSuggestions = (prefix) => {
-      const tableId = this.gristDoc.viewModel.activeSection().table().tableId();
+      const section = this.gristDoc.viewModel.activeSection();
+      const tableId = section.table().tableId();
       const columnId = this.column.colId();
-      return this.gristDoc.docComm.autocomplete(prefix, tableId, columnId);
+      const rowId = section.activeRowId();
+      return this.gristDoc.docComm.autocomplete(prefix, tableId, columnId, rowId);
     };
     setupAceEditorCompletions(this.editor, {getSuggestions});
   }

app/client/components/AceEditorCompletions.ts

@@ -1,14 +1,8 @@
+import {ISuggestionWithValue} from 'app/common/ActiveDocAPI';
 import * as ace from 'brace';
 
-// Suggestion may be a string, or a tuple [funcname, argSpec, isGrist], where:
-//  - funcname (e.g. "DATEADD") will be auto-completed with "(", AND linked to Grist
-//    documentation.
-//  - argSpec (e.g. "(start_date, days=0, ...)") is to be shown as autocomplete caption.
-//  - isGrist determines whether to tag this suggestion as "grist" or "python".
-export type ISuggestion = string | [string, string, boolean];
-
 export interface ICompletionOptions {
-  getSuggestions(prefix: string): Promise<ISuggestion[]>;
+  getSuggestions(prefix: string): Promise<ISuggestionWithValue[]>;
 }
 
 const completionOptions = new WeakMap<ace.Editor, ICompletionOptions>();
@@ -27,18 +21,29 @@ export function setupAceEditorCompletions(editor: ace.Editor, options: ICompleti
   completer.autoSelect = false;
   (editor as any).completer = completer;
 
-  // Patch updateCompletions and insertMatch so that fresh completions are fetched when the user types '.' or '('
+  // Used in the patches below. Returns true if the client should fetch fresh completions from the server,
+  // as it may have new suggestions that aren't currently shown.
+  completer._gristShouldRefreshCompletions = function(this: any, start: any) {
+    // These two lines are based on updateCompletions() in the ace autocomplete source code.
+    const end = this.editor.getCursorPosition();
+    const prefix: string = this.editor.session.getTextRange({start, end}).toLowerCase();
+
+    return (
+      prefix.endsWith(".") ||  // to get fresh attributes of references
+      prefix.endsWith(".lookupone(") ||  // to get initial argument suggestions
+      prefix.endsWith(".lookuprecords(")
+    );
+  }.bind(completer);
+
+  // Patch updateCompletions and insertMatch so that fresh completions are fetched when appropriate.
   const originalUpdate = completer.updateCompletions.bind(completer);
   completer.updateCompletions = function(this: any, keepPopupPosition: boolean) {
-    // The next three lines are copied from updateCompletions() in the ace autocomplete source code.
+    // This next line is copied from updateCompletions() in the ace autocomplete source code.
     if (keepPopupPosition && this.base && this.completions) {
-      const pos = this.editor.getCursorPosition();
+      // When we need fresh completions, prevent this same block from running
-      const prefix = this.editor.session.getTextRange({start: this.base, end: pos});
-      // If the cursor is just after '.' or '(', prevent this same block from running
       // in the original updateCompletions() function. Otherwise it will just keep any remaining completions that match,
       // or not show any completions at all.
-      // But the last character implies that the set of completions is likely to have changed.
+      if (this._gristShouldRefreshCompletions(this.base)) {
-      if (prefix.endsWith(".") || prefix.endsWith("(")) {
         this.completions = null;
       }
     }
@@ -50,12 +55,7 @@ export function setupAceEditorCompletions(editor: ace.Editor, options: ICompleti
   completer.insertMatch = function(this: any) {
     const base = this.base;  // this.base may become null after the next line, save it now.
     const result = originalInsertMatch.apply(...arguments);
-    // Like in the above patch, get the current text in the editor to be completed.
+    if (this._gristShouldRefreshCompletions(base)) {
-    const pos = this.editor.getCursorPosition();
-    const prefix = this.editor.session.getTextRange({start: base, end: pos});
-    // This patch is specifically for when a previous completion is inserted by pressing Enter/Tab,
-    // and such completions may end in '(', which can lead to more completions, e.g. for `.lookupRecords(`.
-    if (prefix.endsWith("(")) {
       this.showPopup(this.editor);
     }
     return result;
@@ -100,19 +100,73 @@ function initCustomCompleter() {
       const suggestions = await options.getSuggestions(prefix);
       // ACE autocompletions are very poorly documented. This is somewhat helpful:
       // https://prog.world/implementing-code-completion-in-ace-editor/
-      callback(null, suggestions.map(suggestion => {
+      const completions: AceSuggestion[] = suggestions.map(suggestionWithValue => {
+        const [suggestion, example] = suggestionWithValue;
         if (Array.isArray(suggestion)) {
-          const [funcname, argSpec, isGrist] = suggestion;
+          const [funcname, argSpec] = suggestion;
-          const meta = isGrist ? 'grist' : 'python';
+          return {
-          return {value: funcname + '(', caption: funcname + argSpec, score: 1, meta, funcname};
+            value: funcname + '(',
+            caption: funcname + argSpec,
+            score: 1,
+            example,
+            funcname,
+          };
         } else {
-          return {value: suggestion, score: 1, meta: "python"};
+          return {
+            value: suggestion,
+            caption: suggestion,
+            score: 1,
+            example,
+            funcname: '',
+          };
         }
-      }));
+      });
+
+      // For suggestions with example values, calculate the 'shared padding', i.e.
+      // the minimum width in characters that all suggestions should fill
+      // (before adding 'base padding') so that the examples are aligned.
+      const captionLengths = completions.filter(c => c.example).map(c => c.caption.length);
+      const sharedPadding = Math.min(
+        Math.min(...captionLengths) + MAX_RELATIVE_SHARED_PADDING,
+        Math.max(...captionLengths),
+        MAX_ABSOLUTE_SHARED_PADDING,
+      );
+
+      // Add the padding spaces and example values to the captions.
+      for (const c of completions) {
+        if (!c.example) { continue; }
+        const numSpaces = Math.max(0, sharedPadding - c.caption.length) + BASE_PADDING;
+        c.caption = c.caption + ' '.repeat(numSpaces) + c.example;
+      }
+
+      callback(null, completions);
     },
   });
 }
 
+// Regardless of other suggestions, always add this many spaces between the caption and the example.
+const BASE_PADDING = 8;
+// In addition to the base padding, there's shared padding, which is the minimum number of spaces
+// that all suggestions should fill so that the examples are aligned.
+// However, one extremely long suggestion shouldn't result in huge padding for all suggestions.
+// To mitigate this, there are two limits on the shared padding.
+// The first limit is relative to the shortest caption in the suggestions.
+// So if all the suggestions are similarly long, there will still be some shared padding.
+const MAX_RELATIVE_SHARED_PADDING = 15;
+// The second limit is absolute, so that even if all suggestions are long, we don't run out of popup space.
+const MAX_ABSOLUTE_SHARED_PADDING = 40;
+
+// Suggestion objects that are passed to ace.
+interface AceSuggestion {
+  value: string;    // the actual value inserted by the autocomplete
+  caption: string;  // the value displayed in the popup
+  score: number;
+
+  // Custom attributes used only by us
+  example: string | null;  // example value of the suggestion to show on the right
+  funcname: string;        // name of a function to link to in documentation
+}
+
 /**
  * When autocompleting a known function (with funcname received from the server call), turn the
  * function name into a link to Grist documentation.
@@ -159,8 +213,8 @@ interface TokenInfo extends ace.TokenInfo {
   type: string;
 }
 
-function retokenizeAceCompleterRow(rowData: any, tokens: TokenInfo[]): TokenInfo[] {
+function retokenizeAceCompleterRow(rowData: AceSuggestion, tokens: TokenInfo[]): TokenInfo[] {
-  if (!rowData.funcname) {
+  if (!(rowData.funcname || rowData.example)) {
     // Not a special completion, pass through the result of ACE's original tokenizing.
     return tokens;
   }
@@ -186,9 +240,33 @@ function retokenizeAceCompleterRow(rowData: any, tokens: TokenInfo[]): TokenInfo
     rowData.funcname.slice(linkStart, linkEnd).toLowerCase();
   newTokens.push({value: href, type: 'grist_link_hidden'});
 
+  // Find where the example value (if any) starts, so that it can be shown in grey.
+  let exampleStart: number | undefined;
+  if (rowData.example) {
+    if (!rowData.caption.endsWith(rowData.example)) {
+      // Just being cautious, this shouldn't happen.
+      console.warn(`Example "${rowData.example}" does not match caption "${rowData.caption}"`);
+    } else {
+      exampleStart = rowData.caption.length - rowData.example.length;
+    }
+  }
+
   // Go through tokens, splitting them if needed, and modifying those that form the link part.
   let position = 0;
   for (const t of tokens) {
+    if (exampleStart && position + t.value.length > exampleStart) {
+      // Ensure that all text after `exampleStart` has the type 'grist_example'.
+      // Don't combine that type with the existing type, because ace highlights weirdly sometimes
+      // and it's best to just override that.
+      const end = exampleStart - position;
+      if (end > 0) {
+        newTokens.push({value: t.value.slice(0, end), type: t.type});
+        newTokens.push({value: t.value.slice(end), type: 'grist_example'});
+      } else {
+        newTokens.push({value: t.value, type: 'grist_example'});
+      }
+    } else {
+      // Handle links to documentation.
       // lStart/lEnd are indices of the link within the token, possibly negative.
       const lStart = linkStart - position, lEnd = linkEnd - position;
       if (lStart > 0) {
@@ -199,11 +277,14 @@ function retokenizeAceCompleterRow(rowData: any, tokens: TokenInfo[]): TokenInfo
         const inLink = t.value.slice(Math.max(0, lStart), lEnd);
         const newType = t.type + (t.type ? '.' : '') + 'grist_link';
         newTokens.push({value: inLink, type: newType});
-    }
         if (lEnd < t.value.length) {
           const afterLink = t.value.slice(lEnd);
           newTokens.push({value: afterLink, type: t.type});
         }
+      } else {
+        newTokens.push(t);
+      }
+    }
     position += t.value.length;
   }
   return newTokens;

app/common/ActiveDocAPI.ts

@@ -1,6 +1,7 @@
 import {ActionGroup} from 'app/common/ActionGroup';
 import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions';
 import {FormulaProperties} from 'app/common/GranularAccessClause';
+import {UIRowId} from 'app/common/UIRowId';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {DocStateComparison, PermissionData, UserAccessData} from 'app/common/UserAPI';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
@@ -178,6 +179,18 @@ export function summaryGroupByDescription(groupByColumnLabels: string[]): string
   return `[${groupByColumnLabels.length ? 'by ' + groupByColumnLabels.join(", ") : "Totals"}]`;
 }
 
+//// Types for autocomplete suggestions
+
+// Suggestion may be a string, or a tuple [funcname, argSpec, isGrist], where:
+//  - funcname (e.g. "DATEADD") will be auto-completed with "(", AND linked to Grist
+//    documentation.
+//  - argSpec (e.g. "(start_date, days=0, ...)") is to be shown as autocomplete caption.
+//  - isGrist is no longer used
+type ISuggestion = string | [string, string, boolean];
+
+// Suggestion paired with an optional example value to show on the right
+export type ISuggestionWithValue = [ISuggestion, string | null];
+
 export interface ActiveDocAPI {
   /**
    * Closes a document, and unsubscribes from its userAction events.
@@ -269,7 +282,7 @@ export interface ActiveDocAPI {
    * Find and return a list of auto-complete suggestions that start with `txt`, when editing a
    * formula in table `tableId` and column `columnId`.
    */
-  autocomplete(txt: string, tableId: string, columnId: string): Promise<string[]>;
+  autocomplete(txt: string, tableId: string, columnId: string, rowId: UIRowId): Promise<ISuggestionWithValue[]>;
 
   /**
    * Removes the current instance from the doc.

app/server/lib/ActiveDoc.ts

@@ -21,6 +21,7 @@ import {
   ForkResult,
   ImportOptions,
   ImportResult,
+  ISuggestionWithValue,
   MergeOptions,
   PermissionDataWithExtraUsers,
   QueryResult,
@@ -64,6 +65,7 @@ import {Interval} from 'app/common/Interval';
 import * as roles from 'app/common/roles';
 import {schema, SCHEMA_VERSION} from 'app/common/schema';
 import {MetaRowRecord, SingleCell} from 'app/common/TableData';
+import {UIRowId} from 'app/common/UIRowId';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
 import {convertFromColumn} from 'app/common/ValueConverter';
@@ -1272,12 +1274,14 @@ export class ActiveDoc extends EventEmitter {
     docSession.linkId = 0;
   }
 
-  public async autocomplete(docSession: DocSession, txt: string, tableId: string, columnId: string): Promise<string[]> {
+  public async autocomplete(
+    docSession: DocSession, txt: string, tableId: string, columnId: string, rowId: UIRowId
+  ): Promise<ISuggestionWithValue[]> {
     // Autocompletion can leak names of tables and columns.
     if (!await this._granularAccess.canScanData(docSession)) { return []; }
     await this.waitForInitialization();
     const user = await this._granularAccess.getCachedUser(docSession);
-    return this._pyCall('autocomplete', txt, tableId, columnId, user.toJSON());
+    return this._pyCall('autocomplete', txt, tableId, columnId, rowId, user.toJSON());
   }
 
   public fetchURL(docSession: DocSession, url: string, options?: FetchUrlOptions): Promise<UploadResult> {

sandbox/grist/autocomplete_context.py

@@ -7,7 +7,7 @@ lowercase searches, and adds function usage information to some results.
 import inspect
 import re
 from collections import namedtuple, defaultdict
-from six.moves import builtins
+from six.moves import builtins, reprlib
 import six
 
 import column
@@ -145,12 +145,14 @@ def lookup_autocomplete_options(lookup_table, formula_table, reverse_only):
     for col_id, col in formula_table.all_columns.items():
       # Note that we can't support reflist columns in the current table,
       # as there is no `IN()` function to do the opposite of the `CONTAINS()` function.
-      if isinstance(col, column.ReferenceColumn) and column.is_user_column(col_id):
+      if isinstance(col, column.ReferenceColumn) and column.is_visible_column(col_id):
         ref_cols[col._target_table].append(col_id)
 
   # Find referencing columns in the lookup table that target tables in ref_cols.
   results = []
   for lookup_col_id, lookup_col in lookup_table.all_columns.items():
+    if not column.is_visible_column(lookup_col_id):
+      continue
     if isinstance(lookup_col, column.ReferenceColumn):
       value_template = "${}"
     elif isinstance(lookup_col, column.ReferenceListColumn):
@@ -162,3 +164,78 @@ def lookup_autocomplete_options(lookup_table, formula_table, reverse_only):
       value = value_template.format(ref_col_id)
       results.append("{}={})".format(lookup_col_id, value))
   return results
+
+
+def eval_suggestion(suggestion, rec, user):
+  """
+  Evaluate a simple string of Python code,
+  and return a limited string representation of the result,
+  or None if this isn't possible.
+  Only supports code starting with `rec` or `user`,
+  followed by any number of attribute accesses, nothing else.
+  """
+
+  if not isinstance(suggestion, six.string_types):
+    # `suggestion` is a tuple corresponding to a function
+    return None
+
+  parts = suggestion.split(".")
+  if parts[0] == "rec":
+    result = rec
+  elif parts[0] == "user":
+    result = user
+    if parts in (["user"], ["user", "LinkKey"]):
+      # `user` and `user.LinkKey` have no useful string representation.
+      return None
+  else:
+    # Other variables are not supported since we can't know their values.
+    return None
+
+  parts = parts[1:]  # attribute names, if any
+  for part in parts:
+    try:
+      result = getattr(result, part)
+    except Exception:
+      return None
+
+  # Convert the value to a string and truncate the length if needed.
+  return repr_example(result)[:arepr.maxother]
+
+
+class AutocompleteExampleRepr(reprlib.Repr):
+  """
+  The default repr for dates and datetimes is long and ugly.
+  This class is used so that repr_example is mostly the same as repr,
+  but dates look the way they're formatted in Grist.
+  """
+  @staticmethod
+  def repr_date(obj, _level):
+    # e.g. "2019-12-31"
+    return obj.strftime("%Y-%m-%d")
+
+  @staticmethod
+  def repr_datetime(obj, _level):
+    # e.g. "2019-12-31 1:23pm"
+    return obj.strftime("%Y-%m-%d %-I:%M%p").lower()
+
+
+arepr = AutocompleteExampleRepr()
+# Set the same high value for all limits, because we just want to avoid
+# sending huge strings to the client, but the truncation shouldn't be visible in the UI.
+arepr.maxother = 200
+arepr.maxtuple = arepr.maxother
+arepr.maxlist = arepr.maxother
+arepr.maxarray = arepr.maxother
+arepr.maxdict = arepr.maxother
+arepr.maxset = arepr.maxother
+arepr.maxfrozenset = arepr.maxother
+arepr.maxdeque = arepr.maxother
+arepr.maxstring = arepr.maxother
+arepr.maxlong = arepr.maxother
+
+def repr_example(x):
+  try:
+    return arepr.repr(x)
+  except Exception:
+    # Copied from Repr.repr_instance in Python 3.
+    return '<%s instance at %#x>' % (x.__class__.__name__, id(x))

sandbox/grist/column.py

@@ -27,18 +27,12 @@ MANUAL_SORT_DEFAULT = 2147483647.0
 
 SPECIAL_COL_IDS = {'id', MANUAL_SORT}
 
-def is_user_column(col_id):
-  """
-  Returns whether the col_id is of a user column (as opposed to special columns that can't be used
-  for user data).
-  """
-  return col_id not in SPECIAL_COL_IDS and not col_id.startswith('#')
 
 def is_visible_column(col_id):
   """
   Returns whether this is an id of a column that's intended to be shown to the user.
   """
-  return is_user_column(col_id) and not col_id.startswith('gristHelper_')
+  return col_id not in SPECIAL_COL_IDS and not col_id.startswith(('#', 'gristHelper_'))
 
 def is_virtual_column(col_id):
   """

sandbox/grist/engine.py

@@ -19,7 +19,7 @@ from sortedcontainers import SortedSet
 import acl
 import actions
 import action_obj
-from autocomplete_context import AutocompleteContext, lookup_autocomplete_options
+from autocomplete_context import AutocompleteContext, lookup_autocomplete_options, eval_suggestion
 from codebuilder import DOLLAR_REGEX
 import depend
 import docactions
@@ -1408,7 +1408,7 @@ class Engine(object):
     if not self._in_update_loop:
       self._bring_mlookups_up_to_date(doc_action)
 
-  def autocomplete(self, txt, table_id, column_id, user):
+  def autocomplete(self, txt, table_id, column_id, row_id, user):
     """
     Return a list of suggested completions of the python fragment supplied.
     """
@@ -1425,13 +1425,15 @@ class Engine(object):
         result = [
           txt + col_id + "="
           for col_id in lookup_table.all_columns
-          if column.is_user_column(col_id) or col_id == 'id'
+          if column.is_visible_column(col_id) or col_id == 'id'
         ]
         # Add specific complete lookups involving reference columns.
         result += [
           txt + option
           for option in lookup_autocomplete_options(lookup_table, table, reverse_only=False)
         ]
+        # Add a dummy empty example value for each result to produce the correct shape.
+        result = [(r, None) for r in result]
         return sorted(result)
 
     # replace $ with rec. and add a dummy rec object
@@ -1479,11 +1481,24 @@ class Engine(object):
             for option in lookup_autocomplete_options(lookup_table, table, reverse_only=True)
           ]
 
+    ### Add example values to all results where possible.
+    if row_id == "new":
+      row_id = table.row_ids.max()
+    rec = table.Record(row_id)
+    # Don't use the same user object as above because we don't want is_sample=True,
+    # which is only needed for the sake of suggesting completions.
+    # Here we want to show actual values.
+    user_obj = User(user, self.tables)
+    results = [
+      (result, eval_suggestion(result, rec, user_obj))
+      for result in results
+    ]
+
     # If we changed the prefix (expanding the $ symbol) we now need to change it back.
     if tweaked_txt != txt:
-      results = [txt + result[len(tweaked_txt):] for result in results]
+      results = [(txt + result[len(tweaked_txt):], value) for result, value in results]
     # pylint:disable=unidiomatic-typecheck
-    results.sort(key=lambda r: r[0] if type(r) == tuple else r)
+    results.sort(key=lambda r: r[0][0] if type(r[0]) == tuple else r[0])
     return results
 
   def _get_undo_checkpoint(self):

sandbox/grist/main.py

@@ -86,8 +86,8 @@ def run(sandbox):
     return eng.fetch_table_schema()
 
   @export
-  def autocomplete(txt, table_id, column_id, user):
+  def autocomplete(txt, table_id, column_id, row_id, user):
-    return eng.autocomplete(txt, table_id, column_id, user)
+    return eng.autocomplete(txt, table_id, column_id, row_id, user)
 
   @export
   def find_col_from_values(values, n, opt_table_id):

sandbox/grist/table.py

@@ -255,7 +255,7 @@ class Table(object):
     # reference values (using .sample_record for other tables) are not yet available.
     props = {}
     for col in self.all_columns.values():
-      if not (column.is_user_column(col.col_id) or col.col_id == 'id'):
+      if not (column.is_visible_column(col.col_id) or col.col_id == 'id'):
         continue
       # Note c=col to bind at lambda-creation time; see
       # https://stackoverflow.com/questions/10452770/python-lambdas-binding-to-local-values

sandbox/grist/test_completion.py

@@ -1,7 +1,11 @@
-import testsamples
+import datetime
+
 import test_engine
+import testsamples
+from autocomplete_context import repr_example, eval_suggestion
 from schema import RecalcWhen
 
+
 class TestCompletion(test_engine.EngineTestCase):
   user = {
     'Name': 'Foo',
@@ -34,85 +38,89 @@ class TestCompletion(test_engine.EngineTestCase):
       type="Text", isFormula=False, formula="foo@getgrist.com",
       recalcWhen=RecalcWhen.MANUAL_UPDATES
     )
+    self.update_record('Schools', 3, budget='123.45', yearFounded='2010', lastModified='2018-01-01')
+    self.update_record('Students', 1, homeAddress=11, school=1)
+    # Create a summary table of Students grouped by school
+    self.apply_user_action(["CreateViewSection", 1, 0, "record", [22], None])
 
   def test_keyword(self):
-    self.assertEqual(self.engine.autocomplete("for", "Address", "city", self.user),
+    self.assertEqual(self.autocomplete("for", "Address", "city"),
                      ["for", "format("])
 
   def test_grist(self):
-    self.assertEqual(self.engine.autocomplete("gri", "Address", "city", self.user),
+    self.assertEqual(self.autocomplete("gri", "Address", "city"),
                      ["grist"])
 
   def test_value(self):
     # Should only appear if column exists and is a trigger formula.
     self.assertEqual(
-      self.engine.autocomplete("val", "Schools", "lastModified", self.user),
+      self.autocomplete("val", "Schools", "lastModified"),
       ["value"]
     )
     self.assertEqual(
-      self.engine.autocomplete("val", "Students", "schoolCities", self.user),
+      self.autocomplete("val", "Students", "schoolCities"),
       []
     )
     self.assertEqual(
-      self.engine.autocomplete("val", "Students", "nonexistentColumn", self.user),
+      self.autocomplete("val", "Students", "nonexistentColumn"),
       []
     )
-    self.assertEqual(self.engine.autocomplete("valu", "Schools", "lastModifier", self.user),
+    self.assertEqual(self.autocomplete("valu", "Schools", "lastModifier"),
                      ["value"])
     # Should have same type as column.
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("value.", "Schools", "lastModifier", self.user)),
+      set(self.autocomplete("value.", "Schools", "lastModifier")),
       {'value.startswith(', 'value.replace(', 'value.title('}
     )
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("value.", "Schools", "lastModified", self.user)),
+      set(self.autocomplete("value.", "Schools", "lastModified")),
       {'value.month', 'value.strftime(', 'value.replace('}
     )
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("value.m", "Schools", "lastModified", self.user)),
+      set(self.autocomplete("value.m", "Schools", "lastModified")),
       {'value.month', 'value.minute'}
     )
 
   def test_user(self):
     # Should only appear if column exists and is a trigger formula.
-    self.assertEqual(self.engine.autocomplete("use", "Schools", "lastModified", self.user),
+    self.assertEqual(self.autocomplete("use", "Schools", "lastModified"),
                      ["user"])
-    self.assertEqual(self.engine.autocomplete("use", "Students", "schoolCities", self.user),
+    self.assertEqual(self.autocomplete("use", "Students", "schoolCities"),
                      [])
-    self.assertEqual(self.engine.autocomplete("use", "Students", "nonexistentColumn", self.user),
+    self.assertEqual(self.autocomplete("use", "Students", "nonexistentColumn"),
                      [])
-    self.assertEqual(self.engine.autocomplete("user", "Schools", "lastModifier", self.user),
+    self.assertEqual(self.autocomplete("user", "Schools", "lastModifier"),
                      ["user"])
     self.assertEqual(
-      self.engine.autocomplete("user.", "Schools", "lastModified", self.user),
+      self.autocomplete("user.", "Schools", "lastModified", row_id=2),
       [
-        'user.Access',
+        ('user.Access', "'owners'"),
-        'user.Email',
+        ('user.Email', "'foo@example.com'"),
-        'user.IsLoggedIn',
+        ('user.IsLoggedIn', 'True'),
-        'user.LinkKey',
+        ('user.LinkKey', None),
-        'user.Name',
+        ('user.Name', "'Foo'"),
-        'user.Origin',
+        ('user.Origin', 'None'),
-        'user.SessionID',
+        ('user.SessionID', "'u1'"),
-        'user.StudentInfo',
+        ('user.StudentInfo', 'Students[1]'),
-        'user.UserID'
+        ('user.UserID', '1'),
       ]
     )
     # Should follow user attribute references and autocomplete those types.
     self.assertEqual(
-      self.engine.autocomplete("user.StudentInfo.", "Schools", "lastModified", self.user),
+      self.autocomplete("user.StudentInfo.", "Schools", "lastModified", row_id=2),
       [
-        'user.StudentInfo.birthDate',
+        ('user.StudentInfo.birthDate', 'None'),
-        'user.StudentInfo.firstName',
+        ('user.StudentInfo.firstName', "'Barack'"),
-        'user.StudentInfo.homeAddress',
+        ('user.StudentInfo.homeAddress', 'Address[11]'),
-        'user.StudentInfo.homeAddress.city',
+        ('user.StudentInfo.homeAddress.city', "'New York'"),
-        'user.StudentInfo.id',
+        ('user.StudentInfo.id', '1'),
-        'user.StudentInfo.lastName',
+        ('user.StudentInfo.lastName', "'Obama'"),
-        'user.StudentInfo.lastVisit',
+        ('user.StudentInfo.lastVisit', 'None'),
-        'user.StudentInfo.school',
+        ('user.StudentInfo.school', 'Schools[1]'),
-        'user.StudentInfo.school.name',
+        ('user.StudentInfo.school.name', "'Columbia'"),
-        'user.StudentInfo.schoolCities',
+        ('user.StudentInfo.schoolCities', repr(u'New York:Colombia')),
-        'user.StudentInfo.schoolIds',
+        ('user.StudentInfo.schoolIds', repr(u'1:2')),
-        'user.StudentInfo.schoolName'
+        ('user.StudentInfo.schoolName', "'Columbia'"),
       ]
     )
     # Should not show user attribute completions if user doesn't have attribute.
@@ -127,27 +135,27 @@ class TestCompletion(test_engine.EngineTestCase):
       'IsLoggedIn': True
     }
     self.assertEqual(
-      self.engine.autocomplete("user.", "Schools", "lastModified", user2),
+      self.autocomplete("user.", "Schools", "lastModified", user2, row_id=2),
       [
-        'user.Access',
+        ('user.Access', "'owners'"),
-        'user.Email',
+        ('user.Email', "'baro@example.com'"),
-        'user.IsLoggedIn',
+        ('user.IsLoggedIn', 'True'),
-        'user.LinkKey',
+        ('user.LinkKey', None),
-        'user.Name',
+        ('user.Name', "'Bar'"),
-        'user.Origin',
+        ('user.Origin', 'None'),
-        'user.SessionID',
+        ('user.SessionID', "'u2'"),
-        'user.UserID'
+        ('user.UserID', '2'),
       ]
     )
     self.assertEqual(
-      self.engine.autocomplete("user.StudentInfo.", "Schools", "schoolCities", user2),
+      self.autocomplete("user.StudentInfo.", "Schools", "schoolCities", user2),
       []
     )
 
   def test_function(self):
-    self.assertEqual(self.engine.autocomplete("MEDI", "Address", "city", self.user),
+    self.assertEqual(self.autocomplete("MEDI", "Address", "city"),
         [('MEDIAN', '(value, *more_values)', True)])
-    self.assertEqual(self.engine.autocomplete("ma", "Address", "city", self.user), [
+    self.assertEqual(self.autocomplete("ma", "Address", "city"), [
       ('MAX', '(value, *more_values)', True),
       ('MAXA', '(value, *more_values)', True),
       'map(',
@@ -156,31 +164,34 @@ class TestCompletion(test_engine.EngineTestCase):
     ])
 
   def test_member(self):
-    self.assertEqual(self.engine.autocomplete("datetime.tz", "Address", "city", self.user),
+    self.assertEqual(self.autocomplete("datetime.tz", "Address", "city"),
                      ["datetime.tzinfo("])
 
   def test_case_insensitive(self):
-    self.assertEqual(self.engine.autocomplete("medi", "Address", "city", self.user),
+    self.assertEqual(self.autocomplete("medi", "Address", "city"),
         [('MEDIAN', '(value, *more_values)', True)])
-    self.assertEqual(self.engine.autocomplete("std", "Address", "city", self.user), [
+    self.assertEqual(self.autocomplete("std", "Address", "city"), [
       ('STDEV', '(value, *more_values)', True),
       ('STDEVA', '(value, *more_values)', True),
       ('STDEVP', '(value, *more_values)', True),
       ('STDEVPA', '(value, *more_values)', True)
     ])
     self.assertEqual(
-      self.engine.autocomplete("stu", "Address", "city", self.user),
+      self.autocomplete("stu", "Address", "city"),
       [
         'Students',
         ('Students.lookupOne', '(colName=<value>, ...)', True),
         ('Students.lookupRecords', '(colName=<value>, ...)', True),
         'Students.lookupRecords(homeAddress=$id)',
+        'Students_summary_school',
+        ('Students_summary_school.lookupOne', '(colName=<value>, ...)', True),
+        ('Students_summary_school.lookupRecords', '(colName=<value>, ...)', True)
       ],
     )
 
     # Add a table name whose lowercase version conflicts with a builtin.
     self.apply_user_action(['AddTable', 'Max', []])
-    self.assertEqual(self.engine.autocomplete("max", "Address", "city", self.user), [
+    self.assertEqual(self.autocomplete("max", "Address", "city"), [
       ('MAX', '(value, *more_values)', True),
       ('MAXA', '(value, *more_values)', True),
       'Max',
@@ -188,7 +199,7 @@ class TestCompletion(test_engine.EngineTestCase):
       ('Max.lookupRecords', '(colName=<value>, ...)', True),
       'max(',
     ])
-    self.assertEqual(self.engine.autocomplete("MAX", "Address", "city", self.user), [
+    self.assertEqual(self.autocomplete("MAX", "Address", "city"), [
       ('MAX', '(value, *more_values)', True),
       ('MAXA', '(value, *more_values)', True),
     ])
@@ -196,23 +207,23 @@ class TestCompletion(test_engine.EngineTestCase):
 
   def test_suggest_globals_and_tables(self):
     # Should suggest globals and table names.
-    self.assertEqual(self.engine.autocomplete("ME", "Address", "city", self.user),
+    self.assertEqual(self.autocomplete("ME", "Address", "city"),
         [('MEDIAN', '(value, *more_values)', True)])
     self.assertEqual(
-      self.engine.autocomplete("Ad", "Address", "city", self.user),
+      self.autocomplete("Ad", "Address", "city"),
       [
         'Address',
         ('Address.lookupOne', '(colName=<value>, ...)', True),
         ('Address.lookupRecords', '(colName=<value>, ...)', True),
       ],
     )
-    self.assertGreaterEqual(set(self.engine.autocomplete("S", "Address", "city", self.user)), {
+    self.assertGreaterEqual(set(self.autocomplete("S", "Address", "city")), {
       'Schools',
       'Students',
       ('SUM', '(value1, *more_values)', True),
       ('STDEV', '(value, *more_values)', True),
     })
-    self.assertGreaterEqual(set(self.engine.autocomplete("s", "Address", "city", self.user)), {
+    self.assertGreaterEqual(set(self.autocomplete("s", "Address", "city")), {
       'Schools',
       'Students',
       'sum(',
@@ -220,7 +231,7 @@ class TestCompletion(test_engine.EngineTestCase):
       ('STDEV', '(value, *more_values)', True),
     })
     self.assertEqual(
-      self.engine.autocomplete("Addr", "Schools", "budget", self.user),
+      self.autocomplete("Addr", "Schools", "budget"),
       [
         'Address',
         ('Address.lookupOne', '(colName=<value>, ...)', True),
@@ -229,26 +240,26 @@ class TestCompletion(test_engine.EngineTestCase):
     )
 
   def test_suggest_columns(self):
-    self.assertEqual(self.engine.autocomplete("$ci", "Address", "city", self.user),
+    self.assertEqual(self.autocomplete("$ci", "Address", "city"),
                      ["$city"])
-    self.assertEqual(self.engine.autocomplete("rec.i", "Address", "city", self.user),
+    self.assertEqual(self.autocomplete("rec.i", "Address", "city"),
                      ["rec.id"])
-    self.assertEqual(len(self.engine.autocomplete("$", "Address", "city", self.user)),
+    self.assertEqual(len(self.autocomplete("$", "Address", "city")),
                      2)
 
     # A few more detailed examples.
-    self.assertEqual(self.engine.autocomplete("$", "Students", "school", self.user),
+    self.assertEqual(self.autocomplete("$", "Students", "school"),
                      ['$birthDate', '$firstName', '$homeAddress', '$homeAddress.city',
                       '$id', '$lastName', '$lastVisit',
                       '$school', '$school.name', '$schoolCities', '$schoolIds', '$schoolName'])
-    self.assertEqual(self.engine.autocomplete("$fi", "Students", "birthDate", self.user),
+    self.assertEqual(self.autocomplete("$fi", "Students", "birthDate"),
                      ['$firstName'])
-    self.assertEqual(self.engine.autocomplete("$school", "Students", "lastVisit", self.user),
+    self.assertEqual(self.autocomplete("$school", "Students", "lastVisit"),
         ['$school', '$school.name', '$schoolCities', '$schoolIds', '$schoolName'])
 
   def test_suggest_lookup_methods(self):
     # Should suggest lookup formulas for tables.
-    address_dot_completion = self.engine.autocomplete("Address.", "Students", "firstName", self.user)
+    address_dot_completion = self.autocomplete("Address.", "Students", "firstName")
     # In python 3.9.7, rlcompleter stops adding parens for property attributes,
     # see https://bugs.python.org/issue44752 - seems like a minor issue, so leave test
     # tolerant.
@@ -262,7 +273,7 @@ class TestCompletion(test_engine.EngineTestCase):
     ])
 
     self.assertEqual(
-      self.engine.autocomplete("Address.lookup", "Students", "lastName", self.user),
+      self.autocomplete("Address.lookup", "Students", "lastName"),
       [
         ('Address.lookupOne', '(colName=<value>, ...)', True),
         ('Address.lookupRecords', '(colName=<value>, ...)', True),
@@ -270,7 +281,7 @@ class TestCompletion(test_engine.EngineTestCase):
     )
 
     self.assertEqual(
-      self.engine.autocomplete("address.look", "Students", "schoolName", self.user),
+      self.autocomplete("address.look", "Students", "schoolName"),
       [
         ('Address.lookupOne', '(colName=<value>, ...)', True),
         ('Address.lookupRecords', '(colName=<value>, ...)', True),
@@ -280,25 +291,25 @@ class TestCompletion(test_engine.EngineTestCase):
   def test_suggest_column_type_methods(self):
     # Should treat columns as correct types.
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("$firstName.", "Students", "firstName", self.user)),
+      set(self.autocomplete("$firstName.", "Students", "firstName")),
       {'$firstName.startswith(', '$firstName.replace(', '$firstName.title('}
     )
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("$birthDate.", "Students", "lastName", self.user)),
+      set(self.autocomplete("$birthDate.", "Students", "lastName")),
       {'$birthDate.month', '$birthDate.strftime(', '$birthDate.replace('}
     )
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("$lastVisit.m", "Students", "firstName", self.user)),
+      set(self.autocomplete("$lastVisit.m", "Students", "firstName")),
       {'$lastVisit.month', '$lastVisit.minute'}
     )
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("$school.", "Students", "firstName", self.user)),
+      set(self.autocomplete("$school.", "Students", "firstName")),
       {'$school.address', '$school.name', '$school.yearFounded', '$school.budget'}
     )
-    self.assertEqual(self.engine.autocomplete("$school.year", "Students", "lastName", self.user),
+    self.assertEqual(self.autocomplete("$school.year", "Students", "lastName"),
                      ['$school.yearFounded'])
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("$yearFounded.", "Schools", "budget", self.user)),
+      set(self.autocomplete("$yearFounded.", "Schools", "budget")),
       {
         '$yearFounded.denominator',    # Only integers have this
         '$yearFounded.bit_length(',    # and this
@@ -306,18 +317,18 @@ class TestCompletion(test_engine.EngineTestCase):
       }
     )
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("$budget.", "Schools", "budget", self.user)),
+      set(self.autocomplete("$budget.", "Schools", "budget")),
       {'$budget.is_integer(', '$budget.real'}    # Only floats have this
     )
 
   def test_suggest_follows_references(self):
     # Should follow references and autocomplete those types.
     self.assertEqual(
-      self.engine.autocomplete("$school.name.st", "Students", "firstName", self.user),
+      self.autocomplete("$school.name.st", "Students", "firstName"),
       ['$school.name.startswith(', '$school.name.strip(']
     )
     self.assertGreaterEqual(
-      set(self.engine.autocomplete("$school.yearFounded.","Students", "firstName", self.user)),
+      set(self.autocomplete("$school.yearFounded.","Students", "firstName")),
       {
         '$school.yearFounded.denominator',
         '$school.yearFounded.bit_length(',
@@ -326,11 +337,11 @@ class TestCompletion(test_engine.EngineTestCase):
     )
 
     self.assertEqual(
-      self.engine.autocomplete("$school.address.", "Students", "lastName", self.user),
+      self.autocomplete("$school.address.", "Students", "lastName"),
       ['$school.address.city', '$school.address.id']
     )
     self.assertEqual(
-      self.engine.autocomplete("$school.address.city.st", "Students", "lastName", self.user),
+      self.autocomplete("$school.address.city.st", "Students", "lastName"),
       ['$school.address.city.startswith(', '$school.address.city.strip(']
     )
 
@@ -339,17 +350,21 @@ class TestCompletion(test_engine.EngineTestCase):
     # including a 'reverse reference' lookup, i.e. `<refcol to current table>=$id`,
     # but only for `lookupRecords`, not `lookupOne`.
     self.assertEqual(
-      self.engine.autocomplete("stu", "Schools", "name", self.user),
+      self.autocomplete("stu", "Schools", "name"),
       [
         'Students',
         ('Students.lookupOne', '(colName=<value>, ...)', True),
         ('Students.lookupRecords', '(colName=<value>, ...)', True),
         # i.e. Students.school is a reference to Schools
         'Students.lookupRecords(school=$id)',
+        'Students_summary_school',
+        ('Students_summary_school.lookupOne', '(colName=<value>, ...)', True),
+        ('Students_summary_school.lookupRecords', '(colName=<value>, ...)', True),
+        'Students_summary_school.lookupRecords(school=$id)',
       ],
     )
     self.assertEqual(
-      self.engine.autocomplete("scho", "Address", "city", self.user),
+      self.autocomplete("scho", "Address", "city"),
       [
         'Schools',
         ('Schools.lookupOne', '(colName=<value>, ...)', True),
@@ -362,7 +377,7 @@ class TestCompletion(test_engine.EngineTestCase):
     # Same as above, but the formula is being entered in 'Students' instead of 'Address',
     # which means there's no reverse reference to suggest.
     self.assertEqual(
-      self.engine.autocomplete("scho", "Students", "firstName", self.user),
+      self.autocomplete("scho", "Students", "firstName"),
       [
         'Schools',
         ('Schools.lookupOne', '(colName=<value>, ...)', True),
@@ -370,11 +385,24 @@ class TestCompletion(test_engine.EngineTestCase):
       ],
     )
 
+    # Test from within a summary table
+    self.assertEqual(
+      self.autocomplete("stu", "Students_summary_school", "count"),
+      [
+        'Students',
+        ('Students.lookupOne', '(colName=<value>, ...)', True),
+        ('Students.lookupRecords', '(colName=<value>, ...)', True),
+        'Students_summary_school',
+        ('Students_summary_school.lookupOne', '(colName=<value>, ...)', True),
+        ('Students_summary_school.lookupRecords', '(colName=<value>, ...)', True),
+      ],
+    )
+
   def test_suggest_lookup_arguments(self):
     # Typing in the full `.lookupRecords(` should suggest keyword argument (i.e. column) names,
     # in addition to reference lookups, including the reverse reference lookups above.
     self.assertEqual(
-      self.engine.autocomplete("Schools.lookupRecords(", "Address", "city", self.user),
+      self.autocomplete("Schools.lookupRecords(", "Address", "city"),
       [
         'Schools.lookupRecords(address=',
         'Schools.lookupRecords(address=$id)',
@@ -391,7 +419,7 @@ class TestCompletion(test_engine.EngineTestCase):
     # columns (one from the looked up table, one from the current table) targeting the same table,
     # e.g. `address=$homeAddress` in the two cases below.
     self.assertEqual(
-      self.engine.autocomplete("Schools.lookupRecords(", "Students", "firstName", self.user),
+      self.autocomplete("Schools.lookupRecords(", "Students", "firstName"),
       [
         'Schools.lookupRecords(address=',
         'Schools.lookupRecords(address=$homeAddress)',
@@ -405,7 +433,7 @@ class TestCompletion(test_engine.EngineTestCase):
     )
 
     self.assertEqual(
-      self.engine.autocomplete("Students.lookupRecords(", "Schools", "name", self.user),
+      self.autocomplete("Students.lookupRecords(", "Schools", "name"),
       [
         'Students.lookupRecords(birthDate=',
         'Students.lookupRecords(firstName=',
@@ -430,7 +458,7 @@ class TestCompletion(test_engine.EngineTestCase):
     # This doesn't affect anything, because there's no way to do the opposite of CONTAINS()
     self.add_column('Schools', 'otherAddresses', type='RefList:Address')
     self.assertEqual(
-      self.engine.autocomplete("Students.lookupRecords(", "Schools", "name", self.user),
+      self.autocomplete("Students.lookupRecords(", "Schools", "name"),
       [
         'Students.lookupRecords(birthDate=',
         'Students.lookupRecords(firstName=',
@@ -453,3 +481,163 @@ class TestCompletion(test_engine.EngineTestCase):
         'Students.lookupRecords(schoolName=',
       ],
     )
+
+  def autocomplete(self, formula, table, column, user=None, row_id=None):
+    """
+    Mild convenience over self.engine.autocomplete.
+    Only returns suggestions without example values, unless row_id is specified.
+    """
+    user = user or self.user
+    results = self.engine.autocomplete(formula, table, column, row_id or 1, user)
+    if row_id is None:
+      return [result for result, value in results]
+    else:
+      return results
+
+  def test_example_values(self):
+    self.assertEqual(
+      self.autocomplete("$", "Schools", "name", row_id=1),
+      [
+        ('$address', 'Address[11]'),
+        ('$budget', '0.0'),
+        ('$id', '1'),
+        ('$lastModified', 'None'),
+        ('$lastModifier', repr(u'')),
+        ('$name', "'Columbia'"),
+        ('$yearFounded', '0'),
+      ],
+    )
+
+    self.assertEqual(
+      self.autocomplete("$", "Schools", "name", row_id=3),
+      [
+        ('$address', 'Address[13]'),
+        ('$budget', '123.45'),
+        ('$id', '3'),
+        ('$lastModified', '2018-01-01 12:00am'),
+        ('$lastModifier', None),
+        ('$name', "'Yale'"),
+        ('$yearFounded', '2010'),
+      ],
+    )
+
+    self.assertEqual(
+      self.autocomplete("$", "Address", "name", row_id=1),
+      [
+        ('$city', repr(u'')),  # for Python 2/3 compatibility
+        ('$id', '0'),  # row_id 1 doesn't exist!
+      ],
+    )
+    self.assertEqual(
+      self.autocomplete("$", "Address", "name", row_id=11),
+      [
+        ('$city', "'New York'"),
+        ('$id', '11'),
+      ],
+    )
+    self.assertEqual(
+      self.autocomplete("$", "Address", "name", row_id='new'),
+      [
+        ('$city', "'West Haven'"),
+        ('$id', '14'),  # row_id 'new' gets replaced with the maximum row ID in the table
+      ],
+    )
+
+    self.assertEqual(
+      self.autocomplete("$", "Students", "name", row_id=1),
+      [
+        ('$birthDate', 'None'),
+        ('$firstName', "'Barack'"),
+        ('$homeAddress', 'Address[11]'),
+        ('$homeAddress.city', "'New York'"),
+        ('$id', '1'),
+        ('$lastName', "'Obama'"),
+        ('$lastVisit', 'None'),
+        ('$school', 'Schools[1]'),
+        ('$school.name', "'Columbia'"),
+        ('$schoolCities', repr(u'New York:Colombia')),
+        ('$schoolIds', repr(u'1:2')),
+        ('$schoolName', "'Columbia'"),
+      ],
+    )
+
+    self.assertEqual(
+      self.autocomplete("rec", "Students", "name", row_id=1),
+      [
+        # Mixture of suggestions with and without values
+        (('RECORD', '(record_or_list, dates_as_iso=False, expand_refs=0)', True), None),
+        ('rec', 'Students[1]'),
+      ],
+    )
+
+  def test_repr(self):
+    date = datetime.date(2019, 12, 31)
+    dtime = datetime.datetime(2019, 12, 31, 13, 23)
+    self.assertEqual(repr_example(date), "2019-12-31")
+    self.assertEqual(repr_example(dtime), "2019-12-31 1:23pm")
+    self.assertEqual(repr_example([1, 'a', dtime, date]),
+                     "[1, 'a', 2019-12-31 1:23pm, 2019-12-31]")
+
+    prefix = "<BadRepr instance at 0x"
+    self.assertEqual(repr_example(BadRepr())[:len(prefix)], prefix)
+
+    big_list = [9] * 100000
+    self.assertEqual(len(big_list), 100000)
+    big_list_repr = repr_example(big_list)
+    self.assertEqual(len(big_list_repr), 605)
+    self.assertEqual(big_list_repr, "[%s...]" % ("9, " * 200))
+
+  def test_eval_suggestion(self):
+    class Record(object):
+      def __init__(self, name):
+        self.name = name
+
+      def __repr__(self):
+        return "Record(%s)" % self.name
+
+      @property
+      def bad(self):
+        raise Exception("bad")
+
+    rec = Record('rec')
+    rec.subrec = Record('subrec')
+    rec.subrec.meaning = 42
+    rec.bad_repr = BadRepr()
+    rec.big = "a" * 100000
+    user = Record('user')
+    user.email = 'my_email'
+    user.LinkKey = Record('LinkKey')
+    user.LinkKey.id = 123
+
+    self.assertEqual(eval_suggestion('rec', rec, user), 'Record(rec)')
+    self.assertEqual(eval_suggestion('rec.subrec', rec, user), 'Record(subrec)')
+    self.assertEqual(eval_suggestion('rec.subrec.meaning', rec, user), '42')
+
+    self.assertEqual(eval_suggestion('rec.spam', rec, user), None)  # doesn't exist
+    self.assertEqual(eval_suggestion('rec.bad', rec, user), None)  # property raises an error
+
+    # attribute exists, but repr() raises an error
+    prefix = "<BadRepr instance at 0x"
+    self.assertEqual(eval_suggestion('rec.bad_repr', rec, user)[:len(prefix)], prefix)
+
+    # attribute exists, but repr() is too long and gets truncated
+    big_repr = repr_example(rec.big)
+    self.assertEqual(eval_suggestion('rec.big', rec, user), big_repr)
+    self.assertEqual(len(big_repr), 200)
+
+    # No string representations for these two
+    self.assertEqual(eval_suggestion('user', rec, user), None)
+    self.assertEqual(eval_suggestion('user.LinkKey', rec, user), None)
+
+    self.assertEqual(eval_suggestion('user.email', rec, user), "'my_email'")
+    self.assertEqual(eval_suggestion('user.LinkKey.id', rec, user), '123')
+
+    self.assertEqual(eval_suggestion('user.spam', rec, user), None)  # doesn't exist
+    self.assertEqual(eval_suggestion('user.bad', rec, user), None)  # property raises an error
+
+    self.assertEqual(eval_suggestion('subrec', rec, user), None)  # other variables not supported
+
+
+class BadRepr(object):
+  def __repr__(self):
+    raise Exception("Bad repr")

sandbox/grist/test_renames.py

@@ -334,15 +334,19 @@ class TestRenames(test_engine.EngineTestCase):
     # Renaming a table should not leave the old name available for auto-complete.
     self.load_sample(self.sample)
     names = {"People", "Persons"}
+    autocomplete = self.engine.autocomplete("Pe", "Address", "city", 1, user)
+    suggestions = {suggestion for suggestion, value in autocomplete}
     self.assertEqual(
-      names.intersection(self.engine.autocomplete("Pe", "Address", "city", user)),
+      names.intersection(suggestions),
       {"People"}
     )
 
     # Rename the table and ensure that "People" is no longer present among top-level names.
-    out_actions = self.apply_user_action(["RenameTable", "People", "Persons"])
+    self.apply_user_action(["RenameTable", "People", "Persons"])
+    autocomplete = self.engine.autocomplete("Pe", "Address", "city", 1, user)
+    suggestions = {suggestion for suggestion, value in autocomplete}
     self.assertEqual(
-      names.intersection(self.engine.autocomplete("Pe", "Address", "city", user)),
+      names.intersection(suggestions),
       {"Persons"}
     )