(core) Formula autocomplete improvements for references and lookups

Summary:
Makes the following improvements to formula autocomplete:

- When a user types `$RefCol` (or part of it), also show `$RefCol.VisibleCol` (replace actual column names) in the autocomplete even before the `.` is typed, to help users understand the difference between a raw reference/record and its visible column.
- When a user types a table name, show `.lookupOne` and `.lookupRecords` in the autocomplete, again even before the `.` is typed.
- For `.lookupRecords(` and `.lookupOne(`, once the `(` is entered, suggest each column name as a keyword argument.
- Also suggest lookup arguments involving compatible reference columns, especially 'reverse reference' lookups like `refcol=$id` which are very common and difficult for users.
- To support these features, the Ace editor autocomplete needs some patching to fetch fresh autocomplete options after typing `.` or `(`. This also improves unrelated behaviour that wasn't great before when one column name is contained in another. See the first added browser test.

Discussions:

- https://grist.slack.com/archives/CDHABLZJT/p1659707068383179
- https://grist.quip.com/HoSmAlvFax0j#MbTADAH5kgG
- https://grist.quip.com/HoSmAlvFax0j/Formula-Improvements#temp:C:MbT3649fe964a184e8dada9bbebb

Test Plan: Added Python and nbrowser tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3580

sandbox/grist/autocomplete_context.py

@@ -5,10 +5,14 @@ It's intended to use with rlcompleter.Completer. It allows finding global names
 lowercase searches, and adds function usage information to some results.
 """
 import inspect
-from collections import namedtuple
+import re
+from collections import namedtuple, defaultdict
 from six.moves import builtins
 import six
 
+import column
+from table import UserTable
+
 # funcname is the function name, e.g. "MAX"
 # argspec is the signature, e.g. "(arg, *more_args)"
 # isgrist is a boolean for whether this function should be in Grist documentation.
@@ -39,17 +43,27 @@ class AutocompleteContext(object):
 
     # Prepare detailed Completion objects for functions where we can supply more info.
     # TODO It would be nice to include builtin functions too, but getargspec doesn't work there.
-    self._functions = {}
+    self._functions = {
+      # Add in the important UserTable methods, with custom friendlier descriptions.
+      '.lookupOne': Completion('.lookupOne', '(colName=<value>, ...)', True),
+      '.lookupRecords': Completion('.lookupRecords', '(colName=<value>, ...)', True),
+      '.Record': Completion('.Record', '', True),
+      '.RecordSet': Completion('.RecordSet', '', True),
+    }
     for key, value in six.iteritems(self._context):
       if value and callable(value):
         argspec = inspect.formatargspec(*inspect.getargspec(value))
         self._functions[key] = Completion(key, argspec, is_grist_func(value))
 
-    # Add in the important UserTable methods, with custom friendlier descriptions.
-    self._functions['.lookupOne'] = Completion('.lookupOne', '(colName=<value>, ...)', True)
-    self._functions['.lookupRecords'] = Completion('.lookupRecords', '(colName=<value>, ...)', True)
-    self._functions['.Record'] = Completion('.Record', '', True)
-    self._functions['.RecordSet'] = Completion('.RecordSet', '', True)
+    for key, value in self._context.copy().items():
+      if isinstance(value, UserTable):
+        for func in [".lookupOne", ".lookupRecords"]:
+          # Add fake variable names like `Table1.lookupOne` to the context.
+          # This allows the method to be suggested
+          # even before the user finishes typing the table name.
+          # Such a variable name isn't actually possible, so it doesn't matter what value we set.
+          self._context[key + func] = None
+          self._functions[key + func] = self._functions[func]._replace(funcname=key + func)
 
     # Remember the original name for each lowercase one.
     self._lowercase = {}
@@ -84,6 +98,15 @@ class AutocompleteContext(object):
     # 'for' suggests the autocompletion 'for ' in python 3
     result = result.rstrip()
 
+    # Table.lookup methods are special to allow completion just from the table name.
+    match = re.search(r'\w+\.(lookupOne|lookupRecords)$', result, re.IGNORECASE)
+    if match:
+      funcname = match.group().lower()
+      funcname = self._lowercase.get(match, funcname)
+      func = self._functions.get(funcname)
+      if func:
+        return tuple(func)
+
     # Callables are returned by rlcompleter with a trailing "(".
     if result.endswith('('):
       funcname = result[0:-1]
@@ -103,3 +126,39 @@ class AutocompleteContext(object):
 
     # Return translation from lowercase if there is one, or the result string otherwise.
     return self._lowercase.get(result, result)
+
+
+def lookup_autocomplete_options(lookup_table, formula_table, reverse_only):
+  """
+  Returns a list of strings to add to `Table.lookupRecords(` (or lookupOne)
+  to suggest arguments for the method.
+  `lookup_table` is the table that the method is being called on.
+  `formula_table` is the table that the formula is being written in.
+  `reverse_only` should be True to only suggest 'reverse reference' lookup arguments
+  (i.e. `<refcol>=$id`) and no other reference lookups (i.e. `<refcol>=$<other refcol>`).
+  """
+  # dict mapping tables to lists of col_ids in `formula_table` that are references
+  # to the the table with that table_id.
+  # In particular `$id` is treated as a reference to `formula_table`.
+  ref_cols = defaultdict(list, {formula_table: ["id"]})
+  if not reverse_only:
+    for col_id, col in formula_table.all_columns.items():
+      # Note that we can't support reflist columns in the current table,
+      # as there is no `IN()` function to do the opposite of the `CONTAINS()` function.
+      if isinstance(col, column.ReferenceColumn) and column.is_user_column(col_id):
+        ref_cols[col._target_table].append(col_id)
+
+  # Find referencing columns in the lookup table that target tables in ref_cols.
+  results = []
+  for lookup_col_id, lookup_col in lookup_table.all_columns.items():
+    if isinstance(lookup_col, column.ReferenceColumn):
+      value_template = "${}"
+    elif isinstance(lookup_col, column.ReferenceListColumn):
+      value_template = "CONTAINS(${})"
+    else:
+      continue
+    target_table_id = lookup_col._target_table
+    for ref_col_id in ref_cols[target_table_id]:
+      value = value_template.format(ref_col_id)
+      results.append("{}={})".format(lookup_col_id, value))
+  return results