(core) Implement PREVIOUS/NEXT/RANK and lookupRecords().find.* methods.

Summary:
- `lookupRecords()` now allows efficient search in sorted results, with
  the syntax  `lookupRecords(..., order_by="-Date").find.le($Date)`. This will find the record with the nearest date that's <= `$Date`.
- The `find.*` methods are `le`, `lt`, `ge`, `gt`, and `eq`. All have O(log N) performance.
- `PREVIOUS(rec, group_by=..., order_by=...)` finds the previous record to rec, according to `group_by` / `order_by`, in amortized O(log N) time. For example, `PREVIOUS(rec, group_by="Account", order_by="Date")`.
- `PREVIOUS(rec, order_by=None)` finds the previous record in the full table, sorted by the `manualSort` column, to match the order visible in the unsorted table.
- `NEXT(...)` is just like `PREVIOUS(...)` but finds the next record.
- `RANK(rec, group_by=..., order_by=..., order="asc")` returns the rank of the record within the group, starting with 1. Order can be `"asc"` (default) or `"desc"`.
- The `order_by` argument in `lookupRecords`, and the new functions now supports tuples, as well as the "-" prefix to reverse order, e.g. `("Category", "-Date")`.
- New functions are only available in Python3, for a minor reason (to support keyword-only arguments for `group_by` and `order_by`) and also as a nudge to Python2 users to update.

- Includes fixes for several situations related to lookups that used to cause quadratic complexity.

Test Plan:
- New performance check that sorted lookups don't add quadratic complexity.
- Tests added for lookup find.* methods, and for PREVIOUS/NEXT/RANK.
- Tests added that renaming columns updates `order_by` and `group_by` arguments, and attributes on results (e.g. `PREVIOUS(...).ColId`) appropriately.
- Python3 tests can now produce verbose output when VERBOSE=1 and -v are given.

Reviewers: jarek, georgegevoian

Reviewed By: jarek, georgegevoian

Subscribers: paulfitz, jarek

Differential Revision: https://phab.getgrist.com/D4265

sandbox/grist/functions/__init__.py

@@ -1,4 +1,6 @@
-# pylint: disable=wildcard-import
+# pylint: disable=wildcard-import, unused-argument
+import six
+
 from .date import *
 from .info import *
 from .logical import *
@@ -8,5 +10,17 @@ from .stats import *
 from .text import *
 from .schedule import *
 
+if six.PY3:
+  # These new functions use Python3-specific syntax.
+  from .prevnext import *   # pylint: disable=import-error
+else:
+  # In Python2, only expose them to guide the user to upgrade.
+  def PREVIOUS(rec, group_by=None, order_by=None):
+    raise NotImplementedError("Update engine to Python3 to use PREVIOUS, NEXT, or RANK")
+  def NEXT(rec, group_by=None, order_by=None):
+    raise NotImplementedError("Update engine to Python3 to use PREVIOUS, NEXT, or RANK")
+  def RANK(rec, group_by=None, order_by=None, order="asc"):
+    raise NotImplementedError("Update engine to Python3 to use PREVIOUS, NEXT, or RANK")
+
 # Export all uppercase names, for use with `from functions import *`.
 __all__ = [k for k in dir() if not k.startswith('_') and k.isupper()]

sandbox/grist/functions/prevnext.py

@@ -0,0 +1,61 @@
+def PREVIOUS(rec, *, group_by=(), order_by):
+  """
+  Finds the previous record in the table according to the order specified by `order_by`, and
+  grouping specified by `group_by`. Each of these arguments may be a column ID or a tuple of
+  column IDs, and `order_by` allows column IDs to be prefixed with "-" to reverse sort order.
+
+  For example,
+  - `PREVIOUS(rec, order_by="Date")` will return the previous record when the list of records is
+    sorted by the Date column.
+  - `PREVIOUS(rec, order_by="-Date")` will return the previous record when the list is sorted by
+    the Date column in descending order.
+  - `PREVIOUS(rec, group_by="Account", order_by="Date")` will return the previous record with the
+    same Account as `rec`, when records are filtered by the Account of `rec` and sorted by Date.
+
+  When multiple records have the same `order_by` values (e.g. the same Date in the examples above),
+  the order is determined by the relative position of rows in views. This is done internally by
+  falling back to the special column `manualSort` and the row ID column `id`.
+
+  Use `order_by=None` to find the previous record in an unsorted table (when rows may be
+  rearranged by dragging them manually). For example,
+  - `PREVIOUS(rec, order_by=None)` will return the previous record in the unsorted list of records.
+
+  You may specify multiple column IDs as a tuple, for both `group_by` and `order_by`. This can be
+  used to match views sorted by multiple columns. For example:
+  - `PREVIOUS(rec, group_by=("Account", "Year"), order_by=("Date", "-Amount"))`
+  """
+  return _sorted_lookup(rec, group_by=group_by, order_by=order_by)._find.previous(rec)
+
+def NEXT(rec, *, group_by=(), order_by):
+  """
+  Finds the next record in the table according to the order specified by `order_by`, and
+  grouping specified by `group_by`. See [`PREVIOUS`](#previous) for details.
+  """
+  return _sorted_lookup(rec, group_by=group_by, order_by=order_by)._find.next(rec)
+
+def RANK(rec, *, group_by=(), order_by, order="asc"):
+  """
+  Returns the rank (or position) of this record in the table according to the order specified by
+  `order_by`, and grouping specified by `group_by`. See [`PREVIOUS`](#previous) for details of
+  these parameters.
+
+  The `order` parameter may be "asc" (which is the default) or "desc".
+
+  When `order` is "asc" or omitted, the first record in the group in the sorted order would have
+  the rank of 1. When `order` is "desc", the last record in the sorted order would have the rank
+  of 1.
+
+  If there are multiple groups, there will be multiple records with the same rank. In particular,
+  each group will have a record with rank 1.
+
+  For example, `RANK(rec, group_by="Year", order_by="Score", order="desc")` will return the rank of
+  the current record (`rec`) among all the records in its table for the same year, ordered by
+  score.
+  """
+  return _sorted_lookup(rec, group_by=group_by, order_by=order_by)._find.rank(rec, order=order)
+
+
+def _sorted_lookup(rec, *, group_by, order_by):
+  if isinstance(group_by, str):
+    group_by = (group_by,)
+  return rec._table.lookup_records(**{c: getattr(rec, c) for c in group_by}, order_by=order_by)

sandbox/grist/functions/stats.py

@@ -495,11 +495,6 @@ def QUARTILE(data, quartile_number):
   """Returns a value nearest to a specified quartile of a dataset."""
   raise NotImplementedError()
 
-@unimplemented
-def RANK(value, data, is_ascending=None):
-  """Returns the rank of a specified value in a dataset."""
-  raise NotImplementedError()
-
 @unimplemented
 def RANK_AVG(value, data, is_ascending=None):
   """Returns the rank of a specified value in a dataset. If there is more than one entry of the same value in the dataset, the average rank of the entries will be returned."""