Summary: - `lookupRecords()` now allows efficient search in sorted results, with the syntax `lookupRecords(..., order_by="-Date").find.le($Date)`. This will find the record with the nearest date that's <= `$Date`. - The `find.*` methods are `le`, `lt`, `ge`, `gt`, and `eq`. All have O(log N) performance. - `PREVIOUS(rec, group_by=..., order_by=...)` finds the previous record to rec, according to `group_by` / `order_by`, in amortized O(log N) time. For example, `PREVIOUS(rec, group_by="Account", order_by="Date")`. - `PREVIOUS(rec, order_by=None)` finds the previous record in the full table, sorted by the `manualSort` column, to match the order visible in the unsorted table. - `NEXT(...)` is just like `PREVIOUS(...)` but finds the next record. - `RANK(rec, group_by=..., order_by=..., order="asc")` returns the rank of the record within the group, starting with 1. Order can be `"asc"` (default) or `"desc"`. - The `order_by` argument in `lookupRecords`, and the new functions now supports tuples, as well as the "-" prefix to reverse order, e.g. `("Category", "-Date")`. - New functions are only available in Python3, for a minor reason (to support keyword-only arguments for `group_by` and `order_by`) and also as a nudge to Python2 users to update. - Includes fixes for several situations related to lookups that used to cause quadratic complexity. Test Plan: - New performance check that sorted lookups don't add quadratic complexity. - Tests added for lookup find.* methods, and for PREVIOUS/NEXT/RANK. - Tests added that renaming columns updates `order_by` and `group_by` arguments, and attributes on results (e.g. `PREVIOUS(...).ColId`) appropriately. - Python3 tests can now produce verbose output when VERBOSE=1 and -v are given. Reviewers: jarek, georgegevoian Reviewed By: jarek, georgegevoian Subscribers: paulfitz, jarek Differential Revision: https://phab.getgrist.com/D4265pull/1112/head
parent
063df75204
commit
f0d0a07295
@ -0,0 +1,61 @@
|
|||||||
|
def PREVIOUS(rec, *, group_by=(), order_by):
|
||||||
|
"""
|
||||||
|
Finds the previous record in the table according to the order specified by `order_by`, and
|
||||||
|
grouping specified by `group_by`. Each of these arguments may be a column ID or a tuple of
|
||||||
|
column IDs, and `order_by` allows column IDs to be prefixed with "-" to reverse sort order.
|
||||||
|
|
||||||
|
For example,
|
||||||
|
- `PREVIOUS(rec, order_by="Date")` will return the previous record when the list of records is
|
||||||
|
sorted by the Date column.
|
||||||
|
- `PREVIOUS(rec, order_by="-Date")` will return the previous record when the list is sorted by
|
||||||
|
the Date column in descending order.
|
||||||
|
- `PREVIOUS(rec, group_by="Account", order_by="Date")` will return the previous record with the
|
||||||
|
same Account as `rec`, when records are filtered by the Account of `rec` and sorted by Date.
|
||||||
|
|
||||||
|
When multiple records have the same `order_by` values (e.g. the same Date in the examples above),
|
||||||
|
the order is determined by the relative position of rows in views. This is done internally by
|
||||||
|
falling back to the special column `manualSort` and the row ID column `id`.
|
||||||
|
|
||||||
|
Use `order_by=None` to find the previous record in an unsorted table (when rows may be
|
||||||
|
rearranged by dragging them manually). For example,
|
||||||
|
- `PREVIOUS(rec, order_by=None)` will return the previous record in the unsorted list of records.
|
||||||
|
|
||||||
|
You may specify multiple column IDs as a tuple, for both `group_by` and `order_by`. This can be
|
||||||
|
used to match views sorted by multiple columns. For example:
|
||||||
|
- `PREVIOUS(rec, group_by=("Account", "Year"), order_by=("Date", "-Amount"))`
|
||||||
|
"""
|
||||||
|
return _sorted_lookup(rec, group_by=group_by, order_by=order_by)._find.previous(rec)
|
||||||
|
|
||||||
|
def NEXT(rec, *, group_by=(), order_by):
|
||||||
|
"""
|
||||||
|
Finds the next record in the table according to the order specified by `order_by`, and
|
||||||
|
grouping specified by `group_by`. See [`PREVIOUS`](#previous) for details.
|
||||||
|
"""
|
||||||
|
return _sorted_lookup(rec, group_by=group_by, order_by=order_by)._find.next(rec)
|
||||||
|
|
||||||
|
def RANK(rec, *, group_by=(), order_by, order="asc"):
|
||||||
|
"""
|
||||||
|
Returns the rank (or position) of this record in the table according to the order specified by
|
||||||
|
`order_by`, and grouping specified by `group_by`. See [`PREVIOUS`](#previous) for details of
|
||||||
|
these parameters.
|
||||||
|
|
||||||
|
The `order` parameter may be "asc" (which is the default) or "desc".
|
||||||
|
|
||||||
|
When `order` is "asc" or omitted, the first record in the group in the sorted order would have
|
||||||
|
the rank of 1. When `order` is "desc", the last record in the sorted order would have the rank
|
||||||
|
of 1.
|
||||||
|
|
||||||
|
If there are multiple groups, there will be multiple records with the same rank. In particular,
|
||||||
|
each group will have a record with rank 1.
|
||||||
|
|
||||||
|
For example, `RANK(rec, group_by="Year", order_by="Score", order="desc")` will return the rank of
|
||||||
|
the current record (`rec`) among all the records in its table for the same year, ordered by
|
||||||
|
score.
|
||||||
|
"""
|
||||||
|
return _sorted_lookup(rec, group_by=group_by, order_by=order_by)._find.rank(rec, order=order)
|
||||||
|
|
||||||
|
|
||||||
|
def _sorted_lookup(rec, *, group_by, order_by):
|
||||||
|
if isinstance(group_by, str):
|
||||||
|
group_by = (group_by,)
|
||||||
|
return rec._table.lookup_records(**{c: getattr(rec, c) for c in group_by}, order_by=order_by)
|
@ -0,0 +1,54 @@
|
|||||||
|
from numbers import Number
|
||||||
|
|
||||||
|
def make_sort_key(table, sort_spec):
|
||||||
|
"""
|
||||||
|
table: Table object from table.py
|
||||||
|
sort_spec: tuple of column IDs, optionally prefixed by '-' to invert the sort order.
|
||||||
|
|
||||||
|
Returns a key class for comparing row_ids, i.e. with the returned SortKey, the expression
|
||||||
|
SortKey(r1) < SortKey(r2) is true iff r1 comes before r2 according to sort_spec.
|
||||||
|
|
||||||
|
The returned SortKey also allows comparing values that aren't in the table:
|
||||||
|
SortKey(row_id, (v1, v2, ...)) will act as if the values of the columns mentioned in
|
||||||
|
sort_spec are v1, v2, etc.
|
||||||
|
"""
|
||||||
|
col_sort_spec = []
|
||||||
|
for col_spec in sort_spec:
|
||||||
|
col_id, sign = (col_spec[1:], -1) if col_spec.startswith('-') else (col_spec, 1)
|
||||||
|
col_obj = table.get_column(col_id)
|
||||||
|
col_sort_spec.append((col_obj, sign))
|
||||||
|
|
||||||
|
class SortKey(object):
|
||||||
|
__slots__ = ("row_id", "values")
|
||||||
|
|
||||||
|
def __init__(self, row_id, values=None):
|
||||||
|
# When values are provided, row_id is not used for access but is used for comparison, so
|
||||||
|
# must still be comparable to any valid row_id (e.g. must not be None). We use
|
||||||
|
# +-sys.float_info.max in records.py for this.
|
||||||
|
self.row_id = row_id
|
||||||
|
self.values = values or tuple(c.get_cell_value(row_id) for (c, _) in col_sort_spec)
|
||||||
|
|
||||||
|
def __lt__(self, other):
|
||||||
|
for (a, b, (col_obj, sign)) in zip(self.values, other.values, col_sort_spec):
|
||||||
|
try:
|
||||||
|
if a < b:
|
||||||
|
return sign == 1
|
||||||
|
if b < a:
|
||||||
|
return sign == -1
|
||||||
|
except TypeError:
|
||||||
|
# Use fallback values to maintain order similar to Python2 (this matches the fallback
|
||||||
|
# logic in SafeSortKey in column.py).
|
||||||
|
# - None is less than everything else
|
||||||
|
# - Numbers are less than other types
|
||||||
|
# - Other types are ordered by type name
|
||||||
|
af = ( (0 if a is None else 1), (0 if isinstance(a, Number) else 1), type(a).__name__ )
|
||||||
|
bf = ( (0 if b is None else 1), (0 if isinstance(b, Number) else 1), type(b).__name__ )
|
||||||
|
if af < bf:
|
||||||
|
return sign == 1
|
||||||
|
if bf < af:
|
||||||
|
return sign == -1
|
||||||
|
|
||||||
|
# Fallback order is by ascending row_id.
|
||||||
|
return self.row_id < other.row_id
|
||||||
|
|
||||||
|
return SortKey
|
@ -0,0 +1,253 @@
|
|||||||
|
import datetime
|
||||||
|
import logging
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import six
|
||||||
|
|
||||||
|
import moment
|
||||||
|
import objtypes
|
||||||
|
import testutil
|
||||||
|
import test_engine
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def D(year, month, day):
|
||||||
|
return moment.date_to_ts(datetime.date(year, month, day))
|
||||||
|
|
||||||
|
class TestLookupFind(test_engine.EngineTestCase):
|
||||||
|
|
||||||
|
def do_setup(self):
|
||||||
|
self.load_sample(testutil.parse_test_sample({
|
||||||
|
"SCHEMA": [
|
||||||
|
[1, "Customers", [
|
||||||
|
[11, "Name", "Text", False, "", "", ""],
|
||||||
|
[12, "MyDate", "Date", False, "", "", ""],
|
||||||
|
]],
|
||||||
|
[2, "Purchases", [
|
||||||
|
[20, "manualSort", "PositionNumber", False, "", "", ""],
|
||||||
|
[21, "Customer", "Ref:Customers", False, "", "", ""],
|
||||||
|
[22, "Date", "Date", False, "", "", ""],
|
||||||
|
[24, "Category", "Text", False, "", "", ""],
|
||||||
|
[25, "Amount", "Numeric", False, "", "", ""],
|
||||||
|
[26, "Prev", "Ref:Purchases", True, "None", "", ""], # To be filled
|
||||||
|
[27, "Cumul", "Numeric", True, "$Prev.Cumul + $Amount", "", ""],
|
||||||
|
]],
|
||||||
|
],
|
||||||
|
"DATA": {
|
||||||
|
"Customers": [
|
||||||
|
["id", "Name", "MyDate"],
|
||||||
|
[1, "Alice", D(2023,12,5)],
|
||||||
|
[2, "Bob", D(2023,12,10)],
|
||||||
|
],
|
||||||
|
"Purchases": [
|
||||||
|
[ "id", "manualSort", "Customer", "Date", "Category", "Amount", ],
|
||||||
|
[1, 1.0, 1, D(2023,12,1), "A", 10],
|
||||||
|
[2, 2.0, 2, D(2023,12,4), "A", 17],
|
||||||
|
[3, 3.0, 1, D(2023,12,3), "A", 20],
|
||||||
|
[4, 4.0, 1, D(2023,12,9), "A", 40],
|
||||||
|
[5, 5.0, 1, D(2023,12,2), "B", 80],
|
||||||
|
[6, 6.0, 1, D(2023,12,6), "B", 160],
|
||||||
|
[7, 7.0, 1, D(2023,12,7), "A", 320],
|
||||||
|
[8, 8.0, 1, D(2023,12,5), "A", 640],
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
|
||||||
|
def do_test_lookup_find(self, find="find", ref_type_to_use=None):
|
||||||
|
self.do_setup()
|
||||||
|
|
||||||
|
if ref_type_to_use:
|
||||||
|
self.add_column("Customers", "PurchasesByDate", type=ref_type_to_use,
|
||||||
|
formula="Purchases.lookupRecords(Customer=$id, sort_by='Date')")
|
||||||
|
lookup = "$PurchasesByDate"
|
||||||
|
else:
|
||||||
|
lookup = "Purchases.lookupRecords(Customer=$id, sort_by='Date')"
|
||||||
|
|
||||||
|
self.add_column("Customers", "LTDate", type="Ref:Purchases",
|
||||||
|
formula="{}.{}.lt($MyDate)".format(lookup, find))
|
||||||
|
self.add_column("Customers", "LEDate", type="Ref:Purchases",
|
||||||
|
formula="{}.{}.le($MyDate)".format(lookup, find))
|
||||||
|
self.add_column("Customers", "GTDate", type="Ref:Purchases",
|
||||||
|
formula="{}.{}.gt($MyDate)".format(lookup, find))
|
||||||
|
self.add_column("Customers", "GEDate", type="Ref:Purchases",
|
||||||
|
formula="{}.{}.ge($MyDate)".format(lookup, find))
|
||||||
|
self.add_column("Customers", "EQDate", type="Ref:Purchases",
|
||||||
|
formula="{}.{}.eq($MyDate)".format(lookup, find))
|
||||||
|
|
||||||
|
# Here's the purchase data sorted by Customer and Date
|
||||||
|
# id Customer Date
|
||||||
|
# 1, 1, D(2023,12,1)
|
||||||
|
# 5, 1, D(2023,12,2)
|
||||||
|
# 3, 1, D(2023,12,3)
|
||||||
|
# 8, 1, D(2023,12,5)
|
||||||
|
# 6, 1, D(2023,12,6)
|
||||||
|
# 7, 1, D(2023,12,7)
|
||||||
|
# 4, 1, D(2023,12,9)
|
||||||
|
# 2, 2, D(2023,12,4)
|
||||||
|
|
||||||
|
# pylint: disable=line-too-long
|
||||||
|
self.assertTableData('Customers', cols="subset", data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,5), LTDate=3, LEDate=8, GTDate=6, GEDate=8, EQDate=8),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,12,10), LTDate=2, LEDate=2, GTDate=0, GEDate=0, EQDate=0),
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Dates for Alice and Bob
|
||||||
|
self.update_record('Customers', 1, MyDate=D(2023,12,4))
|
||||||
|
self.update_record('Customers', 2, MyDate=D(2023,12,4))
|
||||||
|
self.assertTableData('Customers', cols="subset", data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,4), LTDate=3, LEDate=3, GTDate=8, GEDate=8, EQDate=0),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,12,4), LTDate=0, LEDate=2, GTDate=0, GEDate=2, EQDate=2),
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change a Purchase from Alice to Bob, and remove a purchase for Alice
|
||||||
|
self.update_record('Purchases', 5, Customer=2)
|
||||||
|
self.remove_record('Purchases', 3)
|
||||||
|
self.assertTableData('Customers', cols="subset", data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,4), LTDate=1, LEDate=1, GTDate=8, GEDate=8, EQDate=0),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,12,4), LTDate=5, LEDate=2, GTDate=0, GEDate=2, EQDate=2),
|
||||||
|
])
|
||||||
|
|
||||||
|
# Another update to the lookup date for Bob.
|
||||||
|
self.update_record('Customers', 2, MyDate=D(2023,1,1))
|
||||||
|
self.assertTableData('Customers', cols="subset", data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,4), LTDate=1, LEDate=1, GTDate=8, GEDate=8, EQDate=0),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,1,1), LTDate=0, LEDate=0, GTDate=5, GEDate=5, EQDate=0),
|
||||||
|
])
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_lookup_find(self):
|
||||||
|
self.do_test_lookup_find()
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_lookup_underscore_find(self):
|
||||||
|
# Repeat the previous test case with _find in place of find. Normally, we can use
|
||||||
|
# lookupRecords(...).find.*, but if a column named "find" exists, it will shadow this method,
|
||||||
|
# and lookupRecords(...)._find.* may be used instead (with an underscore). Check that it works.
|
||||||
|
self.do_test_lookup_find(find="_find")
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_lookup_find_ref_any(self):
|
||||||
|
self.do_test_lookup_find(ref_type_to_use='Any')
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_lookup_find_ref_reflist(self):
|
||||||
|
self.do_test_lookup_find(ref_type_to_use='RefList:Purchases')
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_lookup_find_empty(self):
|
||||||
|
self.do_setup()
|
||||||
|
self.add_column("Customers", "P", type='RefList:Purchases',
|
||||||
|
formula="Purchases.lookupRecords(Customer=$id, Category='C', sort_by='Date')")
|
||||||
|
self.add_column("Customers", "LTDate", type="Ref:Purchases", formula="$P.find.lt($MyDate)")
|
||||||
|
self.add_column("Customers", "LEDate", type="Ref:Purchases", formula="$P.find.le($MyDate)")
|
||||||
|
self.add_column("Customers", "GTDate", type="Ref:Purchases", formula="$P.find.gt($MyDate)")
|
||||||
|
self.add_column("Customers", "GEDate", type="Ref:Purchases", formula="$P.find.ge($MyDate)")
|
||||||
|
self.add_column("Customers", "EQDate", type="Ref:Purchases", formula="$P.find.eq($MyDate)")
|
||||||
|
|
||||||
|
# pylint: disable=line-too-long
|
||||||
|
self.assertTableData('Customers', cols="subset", data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,5), LTDate=0, LEDate=0, GTDate=0, GEDate=0, EQDate=0),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,12,10), LTDate=0, LEDate=0, GTDate=0, GEDate=0, EQDate=0),
|
||||||
|
])
|
||||||
|
|
||||||
|
# Check find.* results once the lookup result becomes non-empty.
|
||||||
|
self.update_record('Purchases', 5, Category="C")
|
||||||
|
self.assertTableData('Customers', cols="subset", data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,5), LTDate=5, LEDate=5, GTDate=0, GEDate=0, EQDate=0),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,12,10), LTDate=0, LEDate=0, GTDate=0, GEDate=0, EQDate=0),
|
||||||
|
])
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_lookup_find_unsorted(self):
|
||||||
|
self.do_setup()
|
||||||
|
self.add_column("Customers", "P", type='RefList:Purchases',
|
||||||
|
formula="[Purchases.lookupOne(Customer=$id)]")
|
||||||
|
self.add_column("Customers", "LTDate", type="Ref:Purchases", formula="$P.find.lt($MyDate)")
|
||||||
|
err = objtypes.RaisedException(ValueError())
|
||||||
|
self.assertTableData('Customers', cols="subset", data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,5), LTDate=err),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,12,10), LTDate=err),
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY2, "Python 2 only")
|
||||||
|
def test_lookup_find_py2(self):
|
||||||
|
self.do_setup()
|
||||||
|
|
||||||
|
self.add_column("Customers", "LTDate", type="Ref:Purchases",
|
||||||
|
formula="Purchases.lookupRecords(Customer=$id, sort_by='Date').find.lt($MyDate)")
|
||||||
|
|
||||||
|
err = objtypes.RaisedException(NotImplementedError())
|
||||||
|
self.assertTableData('Customers', data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,5), LTDate=err),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,12,10), LTDate=err),
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def test_column_named_find(self):
|
||||||
|
# Test that we can add a column named "find", use it, and remove it.
|
||||||
|
self.do_setup()
|
||||||
|
self.add_column("Customers", "find", type="Text")
|
||||||
|
|
||||||
|
# Check that the column is usable.
|
||||||
|
self.update_record("Customers", 1, find="Hello")
|
||||||
|
self.assertTableData('Customers', cols="all", data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,5), find="Hello"),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,12,10), find=""),
|
||||||
|
])
|
||||||
|
|
||||||
|
# Check that we can remove the column.
|
||||||
|
self.remove_column("Customers", "find")
|
||||||
|
self.assertTableData('Customers', cols="all", data=[
|
||||||
|
dict(id=1, Name="Alice", MyDate=D(2023,12,5)),
|
||||||
|
dict(id=2, Name="Bob", MyDate=D(2023,12,10)),
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_rename_find_attrs(self):
|
||||||
|
"""
|
||||||
|
Check that in formulas like Table.lookupRecords(...).find.lt(...).ColID, renames of ColID
|
||||||
|
update the formula.
|
||||||
|
"""
|
||||||
|
# Create a simple table (People) with a couple records.
|
||||||
|
self.apply_user_action(["AddTable", "People", [
|
||||||
|
dict(id="Name", type="Text")
|
||||||
|
]])
|
||||||
|
self.add_record("People", Name="Alice")
|
||||||
|
self.add_record("People", Name="Bob")
|
||||||
|
|
||||||
|
# Create a separate table that does a lookup in the People table.
|
||||||
|
self.apply_user_action(["AddTable", "Test", [
|
||||||
|
dict(id="Lookup1", type="Any", isFormula=True,
|
||||||
|
formula="People.lookupRecords(order_by='Name').find.ge('B').Name"),
|
||||||
|
dict(id="Lookup2", type="Any", isFormula=True,
|
||||||
|
formula="People.lookupRecords(order_by='Name')._find.eq('Alice').Name"),
|
||||||
|
dict(id="Lookup3", type="Any", isFormula=True,
|
||||||
|
formula="r = People.lookupRecords(order_by='Name').find.ge('B')\n" +
|
||||||
|
"PREVIOUS(r, order_by=None).Name"),
|
||||||
|
dict(id="Lookup4", type="Any", isFormula=True,
|
||||||
|
formula="r = People.lookupRecords(order_by='Name').find.eq('Alice')\n" +
|
||||||
|
"People.lookupRecords(order_by='Name').find.next(r).Name")
|
||||||
|
]])
|
||||||
|
self.add_record("Test")
|
||||||
|
|
||||||
|
# Test that lookups return data as expected.
|
||||||
|
self.assertTableData('Test', cols="subset", data=[
|
||||||
|
dict(id=1, Lookup1="Bob", Lookup2="Alice", Lookup3="Alice", Lookup4="Bob")
|
||||||
|
])
|
||||||
|
|
||||||
|
# Rename a column used for lookups or order_by. Lookup result shouldn't change.
|
||||||
|
self.apply_user_action(["RenameColumn", "People", "Name", "FullName"])
|
||||||
|
self.assertTableData('Test', cols="subset", data=[
|
||||||
|
dict(id=1, Lookup1="Bob", Lookup2="Alice", Lookup3="Alice", Lookup4="Bob")
|
||||||
|
])
|
||||||
|
|
||||||
|
self.assertTableData('_grist_Tables_column', cols="subset", rows="subset", data=[
|
||||||
|
dict(id=6, colId="Lookup3",
|
||||||
|
formula="r = People.lookupRecords(order_by='FullName').find.ge('B')\n" +
|
||||||
|
"PREVIOUS(r, order_by=None).FullName"),
|
||||||
|
dict(id=7, colId="Lookup4",
|
||||||
|
formula="r = People.lookupRecords(order_by='FullName').find.eq('Alice')\n" +
|
||||||
|
"People.lookupRecords(order_by='FullName').find.next(r).FullName")
|
||||||
|
])
|
@ -0,0 +1,115 @@
|
|||||||
|
import math
|
||||||
|
import time
|
||||||
|
import testutil
|
||||||
|
import test_engine
|
||||||
|
|
||||||
|
class TestLookupPerformance(test_engine.EngineTestCase):
|
||||||
|
def test_non_quadratic(self):
|
||||||
|
# This test measures performance which depends on other stuff running on the machine, which
|
||||||
|
# makes it inherently flaky. But if it fails legitimately, it should fail every time. So we
|
||||||
|
# run multiple times (3), and fail only if all of those times fail.
|
||||||
|
for i in range(2):
|
||||||
|
try:
|
||||||
|
return self._do_test_non_quadratic()
|
||||||
|
except Exception as e:
|
||||||
|
print("FAIL #%d" % (i + 1))
|
||||||
|
self._do_test_non_quadratic()
|
||||||
|
|
||||||
|
def _do_test_non_quadratic(self):
|
||||||
|
# If the same lookupRecords is called by many cells, it should reuse calculations, not lead to
|
||||||
|
# quadratic complexity. (Actually making use of the result would often still be O(N) in each
|
||||||
|
# cell, but here we check that just doing the lookup is O(1) amortized.)
|
||||||
|
|
||||||
|
# Table1 has columns: Date and Status, each will have just two distinct values.
|
||||||
|
# We add a bunch of formulas that should take constant time outside of the lookup.
|
||||||
|
|
||||||
|
# The way we test for quadratic complexity is by timing "BulkAddRecord" action that causes all
|
||||||
|
# rows to recalculate for a geometrically growing sequence of row counts. Then we
|
||||||
|
# log-transform the data and do linear regression on it. It should produce data that fits
|
||||||
|
# closely a line of slope 1.
|
||||||
|
|
||||||
|
self.setUp() # Repeat setup because this test case gets called multiple times.
|
||||||
|
self.load_sample(testutil.parse_test_sample({
|
||||||
|
"SCHEMA": [
|
||||||
|
[1, "Table1", [
|
||||||
|
[1, "Date", "Date", False, "", "", ""],
|
||||||
|
[2, "Status", "Text", False, "", "", ""],
|
||||||
|
[3, "lookup_1a", "Any", True, "len(Table1.all)", "", ""],
|
||||||
|
[4, "lookup_2a", "Any", True, "len(Table1.lookupRecords(order_by='-Date'))", "", ""],
|
||||||
|
[5, "lookup_3a", "Any", True,
|
||||||
|
"len(Table1.lookupRecords(Status=$Status, order_by=('-Date', '-id')))", "", ""],
|
||||||
|
[6, "lookup_1b", "Any", True, "Table1.lookupOne().id", "", ""],
|
||||||
|
# Keep one legacy sort_by example (it shares implementation, so should work similarly)
|
||||||
|
[7, "lookup_2b", "Any", True, "Table1.lookupOne(sort_by='-Date').id", "", ""],
|
||||||
|
[8, "lookup_3b", "Any", True,
|
||||||
|
"Table1.lookupOne(Status=$Status, order_by=('-Date', '-id')).id", "", ""],
|
||||||
|
]]
|
||||||
|
],
|
||||||
|
"DATA": {}
|
||||||
|
}))
|
||||||
|
|
||||||
|
num_records = 0
|
||||||
|
|
||||||
|
def add_records(count):
|
||||||
|
assert count % 4 == 0, "Call add_records with multiples of 4 here"
|
||||||
|
self.add_records("Table1", ["Date", "Status"], [
|
||||||
|
[ "2024-01-01", "Green" ],
|
||||||
|
[ "2024-01-01", "Green" ],
|
||||||
|
[ "2024-02-01", "Blue" ],
|
||||||
|
[ "2000-01-01", "Blue" ],
|
||||||
|
] * (count // 4))
|
||||||
|
|
||||||
|
N = num_records + count
|
||||||
|
self.assertTableData(
|
||||||
|
"Table1", cols="subset", rows="subset", data=[
|
||||||
|
["id", "lookup_1a", "lookup_2a", "lookup_3a", "lookup_1b", "lookup_2b", "lookup_3b"],
|
||||||
|
[1, N, N, N // 2, 1, 3, N - 2],
|
||||||
|
])
|
||||||
|
return N
|
||||||
|
|
||||||
|
# Add records in a geometric sequence
|
||||||
|
times = {}
|
||||||
|
start_time = time.time()
|
||||||
|
last_time = start_time
|
||||||
|
count_add = 20
|
||||||
|
while last_time < start_time + 2: # Stop once we've spent 2 seconds
|
||||||
|
add_time = time.time()
|
||||||
|
num_records = add_records(count_add)
|
||||||
|
last_time = time.time()
|
||||||
|
times[num_records] = last_time - add_time
|
||||||
|
count_add *= 2
|
||||||
|
|
||||||
|
count_array = sorted(times.keys())
|
||||||
|
times_array = [times[r] for r in count_array]
|
||||||
|
|
||||||
|
# Perform linear regression on log-transformed data
|
||||||
|
log_count_array = [math.log(x) for x in count_array]
|
||||||
|
log_times_array = [math.log(x) for x in times_array]
|
||||||
|
|
||||||
|
# Calculate slope and intercept using the least squares method.
|
||||||
|
# Doing this manually so that it works in Python2 too.
|
||||||
|
# Otherwise, we could just use statistics.linear_regression()
|
||||||
|
n = len(log_count_array)
|
||||||
|
sum_x = sum(log_count_array)
|
||||||
|
sum_y = sum(log_times_array)
|
||||||
|
sum_xx = sum(x * x for x in log_count_array)
|
||||||
|
sum_xy = sum(x * y for x, y in zip(log_count_array, log_times_array))
|
||||||
|
slope = (n * sum_xy - sum_x * sum_y) / (n * sum_xx - sum_x * sum_x)
|
||||||
|
intercept = (sum_y - slope * sum_x) / n
|
||||||
|
|
||||||
|
# Calculate R-squared
|
||||||
|
mean_y = sum_y / n
|
||||||
|
ss_tot = sum((y - mean_y) ** 2 for y in log_times_array)
|
||||||
|
ss_res = sum((y - (slope * x + intercept)) ** 2
|
||||||
|
for x, y in zip(log_count_array, log_times_array))
|
||||||
|
r_squared = 1 - (ss_res / ss_tot)
|
||||||
|
|
||||||
|
# Check that the slope is close to 1. For log-transformed data, this means a linear
|
||||||
|
# relationship (a quadratic term would make the slope 2).
|
||||||
|
# In practice, we see slope even less 1 (because there is a non-trivial constant term), so we
|
||||||
|
# can assert things a bit lower than 1: 0.86 to 1.04.
|
||||||
|
err_msg = "Time is non-linear: slope {} R^2 {}".format(slope, r_squared)
|
||||||
|
self.assertAlmostEqual(slope, 0.95, delta=0.09, msg=err_msg)
|
||||||
|
|
||||||
|
# Check that R^2 is close to 1, meaning that data is very close to that line (of slope ~1).
|
||||||
|
self.assertAlmostEqual(r_squared, 1, delta=0.08, msg=err_msg)
|
@ -0,0 +1,514 @@
|
|||||||
|
import datetime
|
||||||
|
import logging
|
||||||
|
import moment
|
||||||
|
import testutil
|
||||||
|
import test_engine
|
||||||
|
from table import make_sort_spec
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def D(year, month, day):
|
||||||
|
return moment.date_to_ts(datetime.date(year, month, day))
|
||||||
|
|
||||||
|
class TestLookupSort(test_engine.EngineTestCase):
|
||||||
|
|
||||||
|
def do_setup(self, order_by_arg):
|
||||||
|
self.load_sample(testutil.parse_test_sample({
|
||||||
|
"SCHEMA": [
|
||||||
|
[1, "Customers", [
|
||||||
|
[11, "Name", "Text", False, "", "", ""],
|
||||||
|
[12, "Lookup", "RefList:Purchases", True,
|
||||||
|
"Purchases.lookupRecords(Customer=$id, %s)" % order_by_arg, "", ""],
|
||||||
|
[13, "LookupAmount", "Any", True,
|
||||||
|
"Purchases.lookupRecords(Customer=$id, %s).Amount" % order_by_arg, "", ""],
|
||||||
|
[14, "LookupDotAmount", "Any", True, "$Lookup.Amount", "", ""],
|
||||||
|
[15, "LookupContains", "RefList:Purchases", True,
|
||||||
|
"Purchases.lookupRecords(Customer=$id, Tags=CONTAINS('foo'), %s)" % order_by_arg,
|
||||||
|
"", ""],
|
||||||
|
[16, "LookupContainsDotAmount", "Any", True, "$LookupContains.Amount", "", ""],
|
||||||
|
]],
|
||||||
|
[2, "Purchases", [
|
||||||
|
[21, "Customer", "Ref:Customers", False, "", "", ""],
|
||||||
|
[22, "Date", "Date", False, "", "", ""],
|
||||||
|
[23, "Tags", "ChoiceList", False, "", "", ""],
|
||||||
|
[24, "Category", "Text", False, "", "", ""],
|
||||||
|
[25, "Amount", "Numeric", False, "", "", ""],
|
||||||
|
]],
|
||||||
|
],
|
||||||
|
"DATA": {
|
||||||
|
"Customers": [
|
||||||
|
["id", "Name"],
|
||||||
|
[1, "Alice"],
|
||||||
|
[2, "Bob"],
|
||||||
|
],
|
||||||
|
"Purchases": [
|
||||||
|
[ "id", "Customer", "Date", "Tags", "Category", "Amount", ],
|
||||||
|
# Note: the tenths digit of Amount corresponds to day, for easier ordering of expected
|
||||||
|
# sort results.
|
||||||
|
[1, 1, D(2023,12,1), ["foo"], "A", 10.1],
|
||||||
|
[2, 2, D(2023,12,4), ["foo"], "A", 17.4],
|
||||||
|
[3, 1, D(2023,12,3), ["bar"], "A", 20.3],
|
||||||
|
[4, 1, D(2023,12,9), ["foo", "bar"], "A", 40.9],
|
||||||
|
[5, 1, D(2023,12,2), ["foo", "bar"], "B", 80.2],
|
||||||
|
[6, 1, D(2023,12,6), ["bar"], "B", 160.6],
|
||||||
|
[7, 1, D(2023,12,7), ["foo"], "A", 320.7],
|
||||||
|
[8, 1, D(2023,12,5), ["bar", "foo"], "A", 640.5],
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
|
||||||
|
def test_make_sort_spec(self):
|
||||||
|
"""
|
||||||
|
Test interpretations of different kinds of order_by and sort_by params.
|
||||||
|
"""
|
||||||
|
# Test the default for Table.lookupRecords.
|
||||||
|
self.assertEqual(make_sort_spec(('id',), None, True), ())
|
||||||
|
self.assertEqual(make_sort_spec(('id',), None, False), ())
|
||||||
|
|
||||||
|
# Test legacy sort_by
|
||||||
|
self.assertEqual(make_sort_spec(('Doh',), 'Foo', True), ('Foo',))
|
||||||
|
self.assertEqual(make_sort_spec(None, '-Foo', False), ('-Foo',))
|
||||||
|
|
||||||
|
# Test None, string, tuple, without manualSort.
|
||||||
|
self.assertEqual(make_sort_spec(None, None, False), ())
|
||||||
|
self.assertEqual(make_sort_spec('Bar', None, False), ('Bar',))
|
||||||
|
self.assertEqual(make_sort_spec(('Foo', '-Bar'), None, False), ('Foo', '-Bar'))
|
||||||
|
|
||||||
|
# Test None, string, tuple, WITH manualSort.
|
||||||
|
self.assertEqual(make_sort_spec(None, None, True), ('manualSort',))
|
||||||
|
self.assertEqual(make_sort_spec('Bar', None, True), ('Bar', 'manualSort'))
|
||||||
|
self.assertEqual(make_sort_spec(('Foo', '-Bar'), None, True), ('Foo', '-Bar', 'manualSort'))
|
||||||
|
|
||||||
|
# If 'manualSort' is present, should not be added twice.
|
||||||
|
self.assertEqual(make_sort_spec(('Foo', 'manualSort'), None, True), ('Foo', 'manualSort'))
|
||||||
|
|
||||||
|
# If 'id' is present, fields starting with it are dropped.
|
||||||
|
self.assertEqual(make_sort_spec(('Bar', 'id'), None, True), ('Bar',))
|
||||||
|
self.assertEqual(make_sort_spec(('Foo', 'id', 'manualSort', 'X'), None, True), ('Foo',))
|
||||||
|
self.assertEqual(make_sort_spec('id', None, True), ())
|
||||||
|
|
||||||
|
def test_lookup_sort_by_default(self):
|
||||||
|
"""
|
||||||
|
Tests lookups with default sort (by row_id) using sort_by=None, and how it reacts to changes.
|
||||||
|
"""
|
||||||
|
self.do_setup('sort_by=None')
|
||||||
|
self._do_test_lookup_sort_by_default()
|
||||||
|
|
||||||
|
def test_lookup_order_by_none(self):
|
||||||
|
# order_by=None means default to manualSort. But this test case should not be affected.
|
||||||
|
self.do_setup('order_by=None')
|
||||||
|
self._do_test_lookup_sort_by_default()
|
||||||
|
|
||||||
|
def _do_test_lookup_sort_by_default(self):
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [1, 3, 4, 5, 6, 7, 8],
|
||||||
|
LookupAmount = [10.1, 20.3, 40.9, 80.2, 160.6, 320.7, 640.5],
|
||||||
|
LookupDotAmount = [10.1, 20.3, 40.9, 80.2, 160.6, 320.7, 640.5],
|
||||||
|
LookupContains = [1, 4, 5, 7, 8],
|
||||||
|
LookupContainsDotAmount = [10.1, 40.9, 80.2, 320.7, 640.5],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Customer of Purchase #2 (Bob -> Alice) and check that all got updated.
|
||||||
|
# (The list of purchases for Alice gets the new purchase #2.)
|
||||||
|
out_actions = self.update_record("Purchases", 2, Customer=1)
|
||||||
|
self.assertEqual(out_actions.calls["Customers"], {
|
||||||
|
"Lookup": 2, "LookupAmount": 2, "LookupDotAmount": 2,
|
||||||
|
"LookupContains": 2, "LookupContainsDotAmount": 2,
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [1, 2, 3, 4, 5, 6, 7, 8],
|
||||||
|
LookupAmount = [10.1, 17.4, 20.3, 40.9, 80.2, 160.6, 320.7, 640.5],
|
||||||
|
LookupDotAmount = [10.1, 17.4, 20.3, 40.9, 80.2, 160.6, 320.7, 640.5],
|
||||||
|
LookupContains = [1, 2, 4, 5, 7, 8],
|
||||||
|
LookupContainsDotAmount = [10.1, 17.4, 40.9, 80.2, 320.7, 640.5],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Customer of Purchase #1 (Alice -> Bob) and check that all got updated.
|
||||||
|
# (The list of purchases for Alice loses the purchase #1.)
|
||||||
|
out_actions = self.update_record("Purchases", 1, Customer=2)
|
||||||
|
self.assertEqual(out_actions.calls["Customers"], {
|
||||||
|
"Lookup": 2, "LookupAmount": 2, "LookupDotAmount": 2,
|
||||||
|
"LookupContains": 2, "LookupContainsDotAmount": 2,
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [2, 3, 4, 5, 6, 7, 8],
|
||||||
|
LookupAmount = [17.4, 20.3, 40.9, 80.2, 160.6, 320.7, 640.5],
|
||||||
|
LookupDotAmount = [17.4, 20.3, 40.9, 80.2, 160.6, 320.7, 640.5],
|
||||||
|
LookupContains = [2, 4, 5, 7, 8],
|
||||||
|
LookupContainsDotAmount = [17.4, 40.9, 80.2, 320.7, 640.5],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Date of Purchase #3 to much earlier, and check that all got updated.
|
||||||
|
out_actions = self.update_record("Purchases", 3, Date=D(2023,8,1))
|
||||||
|
# Nothing to recompute in this case, since it doesn't depend on Date.
|
||||||
|
self.assertEqual(out_actions.calls.get("Customers"), None)
|
||||||
|
|
||||||
|
# Change Amount of Purchase #3 to much larger, and check that just amounts got updated.
|
||||||
|
out_actions = self.update_record("Purchases", 3, Amount=999999)
|
||||||
|
self.assertEqual(out_actions.calls["Customers"], {
|
||||||
|
# Lookups that don't depend on Amount aren't recalculated
|
||||||
|
"LookupAmount": 1, "LookupDotAmount": 1,
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [2, 3, 4, 5, 6, 7, 8],
|
||||||
|
LookupAmount = [17.4, 999999, 40.9, 80.2, 160.6, 320.7, 640.5],
|
||||||
|
LookupDotAmount = [17.4, 999999, 40.9, 80.2, 160.6, 320.7, 640.5],
|
||||||
|
LookupContains = [2, 4, 5, 7, 8],
|
||||||
|
LookupContainsDotAmount = [17.4, 40.9, 80.2, 320.7, 640.5],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_lookup_sort_by_date(self):
|
||||||
|
"""
|
||||||
|
Tests lookups with sort by "-Date", and how it reacts to changes.
|
||||||
|
"""
|
||||||
|
self.do_setup('sort_by="-Date"')
|
||||||
|
self._do_test_lookup_sort_by_date()
|
||||||
|
|
||||||
|
def test_lookup_order_by_date(self):
|
||||||
|
# With order_by, we'll fall back to manualSort, but this shouldn't matter here.
|
||||||
|
self.do_setup('order_by="-Date"')
|
||||||
|
self._do_test_lookup_sort_by_date()
|
||||||
|
|
||||||
|
def _do_test_lookup_sort_by_date(self):
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 6, 8, 3, 5, 1],
|
||||||
|
LookupAmount = [40.9, 320.7, 160.6, 640.5, 20.3, 80.2, 10.1],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 160.6, 640.5, 20.3, 80.2, 10.1],
|
||||||
|
LookupContains = [4, 7, 8, 5, 1],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 80.2, 10.1],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Customer of Purchase #2 (Bob -> Alice) and check that all got updated.
|
||||||
|
# (The list of purchases for Alice gets the new purchase #2.)
|
||||||
|
out_actions = self.update_record("Purchases", 2, Customer=1)
|
||||||
|
self.assertEqual(out_actions.calls["Customers"], {
|
||||||
|
"Lookup": 2, "LookupAmount": 2, "LookupDotAmount": 2,
|
||||||
|
"LookupContains": 2, "LookupContainsDotAmount": 2,
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 6, 8, 2, 3, 5, 1],
|
||||||
|
LookupAmount = [40.9, 320.7, 160.6, 640.5, 17.4, 20.3, 80.2, 10.1],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 160.6, 640.5, 17.4, 20.3, 80.2, 10.1],
|
||||||
|
LookupContains = [4, 7, 8, 2, 5, 1],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 17.4, 80.2, 10.1],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Customer of Purchase #1 (Alice -> Bob) and check that all got updated.
|
||||||
|
# (The list of purchases for Alice loses the purchase #1.)
|
||||||
|
out_actions = self.update_record("Purchases", 1, Customer=2)
|
||||||
|
self.assertEqual(out_actions.calls["Customers"], {
|
||||||
|
"Lookup": 2, "LookupAmount": 2, "LookupDotAmount": 2,
|
||||||
|
"LookupContains": 2, "LookupContainsDotAmount": 2,
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 6, 8, 2, 3, 5],
|
||||||
|
LookupAmount = [40.9, 320.7, 160.6, 640.5, 17.4, 20.3, 80.2],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 160.6, 640.5, 17.4, 20.3, 80.2],
|
||||||
|
LookupContains = [4, 7, 8, 2, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 17.4, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Date of Purchase #3 to much earlier, and check that all got updated.
|
||||||
|
out_actions = self.update_record("Purchases", 3, Date=D(2023,8,1))
|
||||||
|
self.assertEqual(out_actions.calls.get("Customers"), {
|
||||||
|
# Only the affected lookups are affected
|
||||||
|
"Lookup": 1, "LookupAmount": 1, "LookupDotAmount": 1
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 6, 8, 2, 5, 3],
|
||||||
|
LookupAmount = [40.9, 320.7, 160.6, 640.5, 17.4, 80.2, 20.3],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 160.6, 640.5, 17.4, 80.2, 20.3],
|
||||||
|
LookupContains = [4, 7, 8, 2, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 17.4, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Amount of Purchase #3 to much larger, and check that just amounts got updated.
|
||||||
|
out_actions = self.update_record("Purchases", 3, Amount=999999)
|
||||||
|
self.assertEqual(out_actions.calls["Customers"], {
|
||||||
|
# Lookups that don't depend on Amount aren't recalculated
|
||||||
|
"LookupAmount": 1, "LookupDotAmount": 1,
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 6, 8, 2, 5, 3],
|
||||||
|
LookupAmount = [40.9, 320.7, 160.6, 640.5, 17.4, 80.2, 999999],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 160.6, 640.5, 17.4, 80.2, 999999],
|
||||||
|
LookupContains = [4, 7, 8, 2, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 17.4, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def test_lookup_order_by_tuple(self):
|
||||||
|
"""
|
||||||
|
Tests lookups with order by ("Category", "-Date"), and how it reacts to changes.
|
||||||
|
"""
|
||||||
|
self.do_setup('order_by=("Category", "-Date")')
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 8, 3, 1, 6, 5],
|
||||||
|
LookupAmount = [40.9, 320.7, 640.5, 20.3, 10.1, 160.6, 80.2],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 640.5, 20.3, 10.1, 160.6, 80.2],
|
||||||
|
LookupContains = [4, 7, 8, 1, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 10.1, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Customer of Purchase #2 (Bob -> Alice) and check that all got updated.
|
||||||
|
# (The list of purchases for Alice gets the new purchase #2.)
|
||||||
|
out_actions = self.update_record("Purchases", 2, Customer=1)
|
||||||
|
self.assertEqual(out_actions.calls["Customers"], {
|
||||||
|
"Lookup": 2, "LookupAmount": 2, "LookupDotAmount": 2,
|
||||||
|
"LookupContains": 2, "LookupContainsDotAmount": 2,
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 8, 2, 3, 1, 6, 5],
|
||||||
|
LookupAmount = [40.9, 320.7, 640.5, 17.4, 20.3, 10.1, 160.6, 80.2],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 640.5, 17.4, 20.3, 10.1, 160.6, 80.2],
|
||||||
|
LookupContains = [4, 7, 8, 2, 1, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 17.4, 10.1, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Customer of Purchase #1 (Alice -> Bob) and check that all got updated.
|
||||||
|
# (The list of purchases for Alice loses the purchase #1.)
|
||||||
|
out_actions = self.update_record("Purchases", 1, Customer=2)
|
||||||
|
self.assertEqual(out_actions.calls["Customers"], {
|
||||||
|
"Lookup": 2, "LookupAmount": 2, "LookupDotAmount": 2,
|
||||||
|
"LookupContains": 2, "LookupContainsDotAmount": 2,
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 8, 2, 3, 6, 5],
|
||||||
|
LookupAmount = [40.9, 320.7, 640.5, 17.4, 20.3, 160.6, 80.2],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 640.5, 17.4, 20.3, 160.6, 80.2],
|
||||||
|
LookupContains = [4, 7, 8, 2, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 17.4, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Date of Purchase #3 to much earlier, and check that all got updated.
|
||||||
|
out_actions = self.update_record("Purchases", 3, Date=D(2023,8,1))
|
||||||
|
self.assertEqual(out_actions.calls.get("Customers"), {
|
||||||
|
# Only the affected lookups are affected
|
||||||
|
"Lookup": 1, "LookupAmount": 1, "LookupDotAmount": 1
|
||||||
|
})
|
||||||
|
# Actually this happens to be unchanged, because within the category, the new date is still in
|
||||||
|
# the same position.
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 8, 2, 3, 6, 5],
|
||||||
|
LookupAmount = [40.9, 320.7, 640.5, 17.4, 20.3, 160.6, 80.2],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 640.5, 17.4, 20.3, 160.6, 80.2],
|
||||||
|
LookupContains = [4, 7, 8, 2, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 17.4, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Category of Purchase #3 to "B", and check that it got moved.
|
||||||
|
out_actions = self.update_record("Purchases", 3, Category="B")
|
||||||
|
self.assertEqual(out_actions.calls.get("Customers"), {
|
||||||
|
# Only the affected lookups are affected
|
||||||
|
"Lookup": 1, "LookupAmount": 1, "LookupDotAmount": 1
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 8, 2, 6, 5, 3],
|
||||||
|
LookupAmount = [40.9, 320.7, 640.5, 17.4, 160.6, 80.2, 20.3],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 640.5, 17.4, 160.6, 80.2, 20.3],
|
||||||
|
LookupContains = [4, 7, 8, 2, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 17.4, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change Amount of Purchase #3 to much larger, and check that just amounts got updated.
|
||||||
|
out_actions = self.update_record("Purchases", 3, Amount=999999)
|
||||||
|
self.assertEqual(out_actions.calls["Customers"], {
|
||||||
|
# Lookups that don't depend on Amount aren't recalculated
|
||||||
|
"LookupAmount": 1, "LookupDotAmount": 1,
|
||||||
|
})
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 8, 2, 6, 5, 3],
|
||||||
|
LookupAmount = [40.9, 320.7, 640.5, 17.4, 160.6, 80.2, 999999],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 640.5, 17.4, 160.6, 80.2, 999999],
|
||||||
|
LookupContains = [4, 7, 8, 2, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 17.4, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_lookup_one(self):
|
||||||
|
self.do_setup('order_by=None')
|
||||||
|
|
||||||
|
# Check that the first value returned by default is the one with the lowest row ID.
|
||||||
|
self.add_column('Customers', 'One', type="Ref:Purchases",
|
||||||
|
formula="Purchases.lookupOne(Customer=$id)")
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(id = 1, Name = "Alice", One = 1),
|
||||||
|
dict(id = 2, Name = "Bob", One = 2),
|
||||||
|
])
|
||||||
|
|
||||||
|
# Check that the first value returned with "-Date" is the one with the highest Date.
|
||||||
|
self.modify_column('Customers', 'One',
|
||||||
|
formula="Purchases.lookupOne(Customer=$id, order_by=('-Date',))")
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(id = 1, Name = "Alice", One = 4),
|
||||||
|
dict(id = 2, Name = "Bob", One = 2),
|
||||||
|
])
|
||||||
|
|
||||||
|
# Check that the first value returned with "-id" is the one with the highest row ID.
|
||||||
|
self.modify_column('Customers', 'One',
|
||||||
|
formula="Purchases.lookupOne(Customer=$id, order_by='-id')")
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(id = 1, Name = "Alice", One = 8),
|
||||||
|
dict(id = 2, Name = "Bob", One = 2),
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def test_renaming_order_by_str(self):
|
||||||
|
# Given some lookups with order_by, rename a column used in order_by. Check order_by got
|
||||||
|
# adjusted, and the results are correct. Try for order_by as string.
|
||||||
|
self.do_setup("order_by='-Date'")
|
||||||
|
self.apply_user_action(['RenameColumn', 'Purchases', 'Category', 'cat'])
|
||||||
|
self.apply_user_action(['RenameColumn', 'Purchases', 'Date', 'Fecha'])
|
||||||
|
|
||||||
|
self.assertTableData('_grist_Tables_column', cols="subset", rows="subset", data=[
|
||||||
|
dict(id=12, colId="Lookup",
|
||||||
|
formula="Purchases.lookupRecords(Customer=$id, order_by='-Fecha')"),
|
||||||
|
dict(id=13, colId="LookupAmount",
|
||||||
|
formula="Purchases.lookupRecords(Customer=$id, order_by='-Fecha').Amount"),
|
||||||
|
])
|
||||||
|
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 6, 8, 3, 5, 1],
|
||||||
|
LookupAmount = [40.9, 320.7, 160.6, 640.5, 20.3, 80.2, 10.1],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 160.6, 640.5, 20.3, 80.2, 10.1],
|
||||||
|
LookupContains = [4, 7, 8, 5, 1],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 80.2, 10.1],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change the (renamed) Date of Purchase #1 to much later, and check that all got updated.
|
||||||
|
self.update_record("Purchases", 1, Fecha=D(2024,12,31))
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [1, 4, 7, 6, 8, 3, 5],
|
||||||
|
LookupAmount = [10.1, 40.9, 320.7, 160.6, 640.5, 20.3, 80.2],
|
||||||
|
LookupDotAmount = [10.1, 40.9, 320.7, 160.6, 640.5, 20.3, 80.2],
|
||||||
|
LookupContains = [1, 4, 7, 8, 5],
|
||||||
|
LookupContainsDotAmount = [10.1, 40.9, 320.7, 640.5, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def test_renaming_order_by_tuple(self):
|
||||||
|
# Given some lookups with order_by, rename a column used in order_by. Check order_by got
|
||||||
|
# adjusted, and the results are correct. Try for order_by as tuple.
|
||||||
|
self.do_setup("order_by=('Category', '-Date')")
|
||||||
|
|
||||||
|
out_actions = self.apply_user_action(['RenameColumn', 'Purchases', 'Category', 'cat'])
|
||||||
|
|
||||||
|
# Check returned actions to ensure we don't produce actions for any stale lookup helper columns
|
||||||
|
# (this is a way to check that we don't forget to clean up stale lookup helper columns).
|
||||||
|
# pylint: disable=line-too-long
|
||||||
|
self.assertPartialOutActions(out_actions, {
|
||||||
|
"stored": [
|
||||||
|
["RenameColumn", "Purchases", "Category", "cat"],
|
||||||
|
["ModifyColumn", "Customers", "Lookup", {"formula": "Purchases.lookupRecords(Customer=$id, order_by=('cat', '-Date'))"}],
|
||||||
|
["ModifyColumn", "Customers", "LookupAmount", {"formula": "Purchases.lookupRecords(Customer=$id, order_by=('cat', '-Date')).Amount"}],
|
||||||
|
["ModifyColumn", "Customers", "LookupContains", {"formula": "Purchases.lookupRecords(Customer=$id, Tags=CONTAINS('foo'), order_by=('cat', '-Date'))"}],
|
||||||
|
["BulkUpdateRecord", "_grist_Tables_column", [24, 12, 13, 15], {"colId": ["cat", "Lookup", "LookupAmount", "LookupContains"], "formula": [
|
||||||
|
"",
|
||||||
|
"Purchases.lookupRecords(Customer=$id, order_by=('cat', '-Date'))",
|
||||||
|
"Purchases.lookupRecords(Customer=$id, order_by=('cat', '-Date')).Amount",
|
||||||
|
"Purchases.lookupRecords(Customer=$id, Tags=CONTAINS('foo'), order_by=('cat', '-Date'))",
|
||||||
|
]}],
|
||||||
|
]
|
||||||
|
})
|
||||||
|
|
||||||
|
self.apply_user_action(['RenameColumn', 'Purchases', 'Date', 'Fecha'])
|
||||||
|
|
||||||
|
self.assertTableData('_grist_Tables_column', cols="subset", rows="subset", data=[
|
||||||
|
dict(id=12, colId="Lookup",
|
||||||
|
formula="Purchases.lookupRecords(Customer=$id, order_by=('cat', '-Fecha'))"),
|
||||||
|
dict(id=13, colId="LookupAmount",
|
||||||
|
formula="Purchases.lookupRecords(Customer=$id, order_by=('cat', '-Fecha')).Amount"),
|
||||||
|
])
|
||||||
|
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 8, 3, 1, 6, 5],
|
||||||
|
LookupAmount = [40.9, 320.7, 640.5, 20.3, 10.1, 160.6, 80.2],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 640.5, 20.3, 10.1, 160.6, 80.2],
|
||||||
|
LookupContains = [4, 7, 8, 1, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 10.1, 80.2],
|
||||||
|
)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Change the (renamed) Date of Purchase #3 to much earlier, and check that all got updated.
|
||||||
|
self.update_record("Purchases", 3, Fecha=D(2023,8,1))
|
||||||
|
self.assertTableData("Customers", cols="subset", rows="subset", data=[
|
||||||
|
dict(
|
||||||
|
id = 1,
|
||||||
|
Name = "Alice",
|
||||||
|
Lookup = [4, 7, 8, 1, 3, 6, 5],
|
||||||
|
LookupAmount = [40.9, 320.7, 640.5, 10.1, 20.3, 160.6, 80.2],
|
||||||
|
LookupDotAmount = [40.9, 320.7, 640.5, 10.1, 20.3, 160.6, 80.2],
|
||||||
|
LookupContains = [4, 7, 8, 1, 5],
|
||||||
|
LookupContainsDotAmount = [40.9, 320.7, 640.5, 10.1, 80.2],
|
||||||
|
)
|
||||||
|
])
|
@ -0,0 +1,389 @@
|
|||||||
|
import datetime
|
||||||
|
import functools
|
||||||
|
import itertools
|
||||||
|
import logging
|
||||||
|
import unittest
|
||||||
|
import six
|
||||||
|
|
||||||
|
import actions
|
||||||
|
from column import SafeSortKey
|
||||||
|
import moment
|
||||||
|
import objtypes
|
||||||
|
import testutil
|
||||||
|
import test_engine
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def D(year, month, day):
|
||||||
|
return moment.date_to_ts(datetime.date(year, month, day))
|
||||||
|
|
||||||
|
|
||||||
|
class TestPrevNext(test_engine.EngineTestCase):
|
||||||
|
|
||||||
|
def do_setup(self):
|
||||||
|
self.load_sample(testutil.parse_test_sample({
|
||||||
|
"SCHEMA": [
|
||||||
|
[1, "Customers", [
|
||||||
|
[11, "Name", "Text", False, "", "", ""],
|
||||||
|
]],
|
||||||
|
[2, "Purchases", [
|
||||||
|
[20, "manualSort", "PositionNumber", False, "", "", ""],
|
||||||
|
[21, "Customer", "Ref:Customers", False, "", "", ""],
|
||||||
|
[22, "Date", "Date", False, "", "", ""],
|
||||||
|
[24, "Category", "Text", False, "", "", ""],
|
||||||
|
[25, "Amount", "Numeric", False, "", "", ""],
|
||||||
|
[26, "Prev", "Ref:Purchases", True, "None", "", ""], # To be filled
|
||||||
|
[27, "Cumul", "Numeric", True, "$Prev.Cumul + $Amount", "", ""],
|
||||||
|
]],
|
||||||
|
],
|
||||||
|
"DATA": {
|
||||||
|
"Customers": [
|
||||||
|
["id", "Name"],
|
||||||
|
[1, "Alice"],
|
||||||
|
[2, "Bob"],
|
||||||
|
],
|
||||||
|
"Purchases": [
|
||||||
|
[ "id", "manualSort", "Customer", "Date", "Category", "Amount", ],
|
||||||
|
[1, 1.0, 1, D(2023,12,1), "A", 10],
|
||||||
|
[2, 2.0, 2, D(2023,12,4), "A", 17],
|
||||||
|
[3, 3.0, 1, D(2023,12,3), "A", 20],
|
||||||
|
[4, 4.0, 1, D(2023,12,9), "A", 40],
|
||||||
|
[5, 5.0, 1, D(2023,12,2), "B", 80],
|
||||||
|
[6, 6.0, 1, D(2023,12,6), "B", 160],
|
||||||
|
[7, 7.0, 1, D(2023,12,7), "A", 320],
|
||||||
|
[8, 8.0, 1, D(2023,12,5), "A", 640],
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
|
||||||
|
def calc_expected(self, group_key=None, sort_key=None, sort_reverse=False):
|
||||||
|
# Returns expected {id, Prev, Cumul} values from Purchases table calculated according to the
|
||||||
|
# given grouping and sorting parameters.
|
||||||
|
group_key = group_key or (lambda r: 0)
|
||||||
|
data = list(actions.transpose_bulk_action(self.engine.fetch_table('Purchases')))
|
||||||
|
expected = []
|
||||||
|
sorted_data = sorted(data, key=sort_key, reverse=sort_reverse)
|
||||||
|
sorted_data = sorted(sorted_data, key=group_key)
|
||||||
|
for key, group in itertools.groupby(sorted_data, key=group_key):
|
||||||
|
prev = 0
|
||||||
|
cumul = 0.0
|
||||||
|
for r in group:
|
||||||
|
cumul = round(cumul + r.Amount, 2)
|
||||||
|
expected.append({"id": r.id, "Prev": prev, "Cumul": cumul})
|
||||||
|
prev = r.id
|
||||||
|
expected.sort(key=lambda r: r["id"])
|
||||||
|
return expected
|
||||||
|
|
||||||
|
def do_test(self, formula, group_key=None, sort_key=None, sort_reverse=False):
|
||||||
|
calc_expected = lambda: self.calc_expected(
|
||||||
|
group_key=group_key, sort_key=sort_key, sort_reverse=sort_reverse)
|
||||||
|
|
||||||
|
def assertPrevValid():
|
||||||
|
# Check that Prev column is legitimate values, e.g. not errors.
|
||||||
|
prev = self.engine.fetch_table('Purchases').columns["Prev"]
|
||||||
|
self.assertTrue(is_all_ints(prev), "Prev column contains invalid values: %s" %
|
||||||
|
[objtypes.encode_object(x) for x in prev])
|
||||||
|
|
||||||
|
# This verification works as follows:
|
||||||
|
# (1) Set "Prev" column to the specified formula.
|
||||||
|
# (2) Calculate expected values for "Prev" and "Cumul" manually, and compare to reality.
|
||||||
|
# (3) Try a few actions that affect the data, and calculate again.
|
||||||
|
self.do_setup()
|
||||||
|
self.modify_column('Purchases', 'Prev', formula=formula)
|
||||||
|
|
||||||
|
# Check the initial data.
|
||||||
|
assertPrevValid()
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected())
|
||||||
|
|
||||||
|
# Check the result after removing a record.
|
||||||
|
self.remove_record('Purchases', 6)
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected())
|
||||||
|
|
||||||
|
# Check the result after updating a record
|
||||||
|
self.update_record('Purchases', 5, Amount=1080) # original value +1000
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected())
|
||||||
|
|
||||||
|
first_date = D(2023, 8, 1)
|
||||||
|
|
||||||
|
# Update a few other records
|
||||||
|
self.update_record("Purchases", 2, Customer=1)
|
||||||
|
self.update_record("Purchases", 1, Customer=2)
|
||||||
|
self.update_record("Purchases", 3, Date=first_date) # becomes earliest in date order
|
||||||
|
assertPrevValid()
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected())
|
||||||
|
|
||||||
|
# Check the result after re-adding a record
|
||||||
|
# Note that Date here matches new date of record #3. This tests sort fallback to rowId.
|
||||||
|
# Amount is the original amount +1.
|
||||||
|
self.add_record('Purchases', 6, manualSort=6.0, Date=first_date, Amount=161)
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected())
|
||||||
|
|
||||||
|
# Update the manualSort value to test how it affects sort results.
|
||||||
|
self.update_record('Purchases', 6, manualSort=0.5)
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected())
|
||||||
|
assertPrevValid()
|
||||||
|
|
||||||
|
def do_test_prevnext(self, formula, group_key=None, sort_key=None, sort_reverse=False):
|
||||||
|
# Run do_test() AND also repeat it after replacing PREVIOUS with NEXT in formula, and
|
||||||
|
# reversing the expected results.
|
||||||
|
|
||||||
|
# Note that this is a bit fragile: it relies on do_test() being limited to only the kinds of
|
||||||
|
# changes that would be reset by another call to self.load_sample().
|
||||||
|
|
||||||
|
with self.subTest(formula=formula): # pylint: disable=no-member
|
||||||
|
self.do_test(formula, group_key=group_key, sort_key=sort_key, sort_reverse=sort_reverse)
|
||||||
|
|
||||||
|
nformula = formula.replace('PREVIOUS', 'NEXT')
|
||||||
|
with self.subTest(formula=nformula): # pylint: disable=no-member
|
||||||
|
self.do_test(nformula, group_key=group_key, sort_key=sort_key, sort_reverse=not sort_reverse)
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_none(self):
|
||||||
|
self.do_test_prevnext("PREVIOUS(rec, order_by=None)", group_key=None,
|
||||||
|
sort_key=lambda r: r.manualSort)
|
||||||
|
|
||||||
|
# Check that order_by arg is required (get TypeError without it).
|
||||||
|
with self.assertRaisesRegex(AssertionError, r'Prev column contains invalid values:.*TypeError'):
|
||||||
|
self.do_test("PREVIOUS(rec)", sort_key=lambda r: -r.id)
|
||||||
|
|
||||||
|
# These assertions are just to ensure that do_test() tests do exercise the feature being
|
||||||
|
# tested, i.e. fail when comparisons are NOT correct.
|
||||||
|
with self.assertRaisesRegex(AssertionError, r'Observed data not as expected'):
|
||||||
|
self.do_test("PREVIOUS(rec, order_by=None)", sort_key=lambda r: -r.id)
|
||||||
|
with self.assertRaisesRegex(AssertionError, r'Observed data not as expected'):
|
||||||
|
self.do_test("PREVIOUS(rec, order_by=None)", group_key=(lambda r: r.Customer),
|
||||||
|
sort_key=(lambda r: r.id))
|
||||||
|
|
||||||
|
# Make sure the test case above exercises the disambiguation by 'manualSort' (i.e. fails if
|
||||||
|
# 'manualSort' isn't used to disambiguate).
|
||||||
|
with self.assertRaisesRegex(AssertionError, r'Observed data not as expected'):
|
||||||
|
self.do_test("PREVIOUS(rec, order_by=None)", sort_key=lambda r: r.id)
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_date(self):
|
||||||
|
self.do_test_prevnext("PREVIOUS(rec, order_by='Date')",
|
||||||
|
group_key=None, sort_key=lambda r: (SafeSortKey(r.Date), r.manualSort))
|
||||||
|
|
||||||
|
# Make sure the test case above exercises the disambiguation by 'manualSort' (i.e. fails if it
|
||||||
|
# isn't used to disambiguate).
|
||||||
|
with self.assertRaisesRegex(AssertionError, r'Observed data not as expected'):
|
||||||
|
self.do_test("PREVIOUS(rec, order_by='Date')",
|
||||||
|
group_key=None, sort_key=lambda r: (SafeSortKey(r.Date), r.id))
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_date_manualsort(self):
|
||||||
|
# Same as the previous test case (with just 'Date'), but specifies 'manualSort' explicitly.
|
||||||
|
self.do_test_prevnext("PREVIOUS(rec, order_by=('Date', 'manualSort'))",
|
||||||
|
group_key=None, sort_key=lambda r: (SafeSortKey(r.Date), r.manualSort))
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_rdate(self):
|
||||||
|
self.do_test_prevnext("PREVIOUS(rec, order_by='-Date')",
|
||||||
|
group_key=None, sort_key=lambda r: (SafeSortKey(r.Date), -r.manualSort), sort_reverse=True)
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_rdate_id(self):
|
||||||
|
self.do_test_prevnext("PREVIOUS(rec, order_by=('-Date', 'id'))",
|
||||||
|
group_key=None, sort_key=lambda r: (SafeSortKey(r.Date), -r.id), sort_reverse=True)
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_customer_rdate(self):
|
||||||
|
self.do_test_prevnext("PREVIOUS(rec, group_by=('Customer',), order_by='-Date')",
|
||||||
|
group_key=(lambda r: r.Customer), sort_key=lambda r: (SafeSortKey(r.Date), -r.id),
|
||||||
|
sort_reverse=True)
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_category_date(self):
|
||||||
|
self.do_test_prevnext("PREVIOUS(rec, group_by=('Category',), order_by='Date')",
|
||||||
|
group_key=(lambda r: r.Category), sort_key=lambda r: SafeSortKey(r.Date))
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_category_date2(self):
|
||||||
|
self.do_test_prevnext("PREVIOUS(rec, group_by='Category', order_by='Date')",
|
||||||
|
group_key=(lambda r: r.Category), sort_key=lambda r: SafeSortKey(r.Date))
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_n_cat_date(self):
|
||||||
|
self.do_test_prevnext("PREVIOUS(rec, order_by=('Category', 'Date'))",
|
||||||
|
sort_key=lambda r: (SafeSortKey(r.Category), SafeSortKey(r.Date)))
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY2, "Python 2 only")
|
||||||
|
def test_prevnext_py2(self):
|
||||||
|
# On Python2, we expect NEXT/PREVIOUS to raise a NotImplementedError. It's not hard to make
|
||||||
|
# it work, but the stricter argument syntax supported by Python3 is helpful, and we'd like
|
||||||
|
# to drop Python2 support anyway.
|
||||||
|
self.do_setup()
|
||||||
|
self.modify_column('Purchases', 'Prev', formula='PREVIOUS(rec, order_by=None)')
|
||||||
|
self.add_column('Purchases', 'Next', formula="NEXT(rec, group_by='Category', order_by='Date')")
|
||||||
|
self.add_column('Purchases', 'Rank', formula="RANK(rec, order_by='Date', order='desc')")
|
||||||
|
|
||||||
|
# Check that all values are the expected exception.
|
||||||
|
err = objtypes.RaisedException(NotImplementedError())
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=[
|
||||||
|
dict(id=r, Prev=err, Next=err, Rank=err, Cumul=err) for r in range(1, 9)
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def do_test_renames(self, formula, renamed_formula, calc_expected_pre, calc_expected_post):
|
||||||
|
self.do_setup()
|
||||||
|
self.modify_column('Purchases', 'Prev', formula=formula)
|
||||||
|
|
||||||
|
# Check the initial data.
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected_pre())
|
||||||
|
|
||||||
|
# Do the renames
|
||||||
|
self.apply_user_action(['RenameColumn', 'Purchases', 'Category', 'cat'])
|
||||||
|
self.apply_user_action(['RenameColumn', 'Purchases', 'Date', 'Fecha'])
|
||||||
|
self.apply_user_action(['RenameColumn', 'Purchases', 'Customer', 'person'])
|
||||||
|
|
||||||
|
# Check that rename worked.
|
||||||
|
self.assertTableData('_grist_Tables_column', cols="subset", rows="subset", data=[
|
||||||
|
dict(id=26, colId="Prev", formula=renamed_formula)
|
||||||
|
])
|
||||||
|
|
||||||
|
# Check that data is as expected, and reacts to changes.
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected_post())
|
||||||
|
|
||||||
|
self.update_record("Purchases", 1, cat="B")
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected_post())
|
||||||
|
|
||||||
|
self.update_record("Purchases", 3, Fecha=D(2023,8,1))
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=calc_expected_post())
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_renaming_prev_str(self):
|
||||||
|
self.do_test_renaming_prevnext_str("PREVIOUS")
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_renaming_next_str(self):
|
||||||
|
self.do_test_renaming_prevnext_str("NEXT")
|
||||||
|
|
||||||
|
def do_test_renaming_prevnext_str(self, func):
|
||||||
|
# Given some PREVIOUS/NEXT calls with group_by and order_by, rename columns mentioned there,
|
||||||
|
# and check columns get adjusted and data remains correct.
|
||||||
|
formula = "{}(rec, group_by='Category', order_by='Date')".format(func)
|
||||||
|
renamed_formula = "{}(rec, group_by='cat', order_by='Fecha')".format(func)
|
||||||
|
self.do_test_renames(formula, renamed_formula,
|
||||||
|
calc_expected_pre = functools.partial(self.calc_expected,
|
||||||
|
group_key=(lambda r: r.Category), sort_key=lambda r: SafeSortKey(r.Date),
|
||||||
|
sort_reverse=(func == 'NEXT')
|
||||||
|
),
|
||||||
|
calc_expected_post = functools.partial(self.calc_expected,
|
||||||
|
group_key=(lambda r: r.cat), sort_key=lambda r: SafeSortKey(r.Fecha),
|
||||||
|
sort_reverse=(func == 'NEXT')
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_renaming_prev_tuple(self):
|
||||||
|
self.do_test_renaming_prevnext_tuple('PREVIOUS')
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_renaming_next_tuple(self):
|
||||||
|
self.do_test_renaming_prevnext_tuple('NEXT')
|
||||||
|
|
||||||
|
def do_test_renaming_prevnext_tuple(self, func):
|
||||||
|
formula = "{}(rec, group_by=('Customer',), order_by=('Category', '-Date'))".format(func)
|
||||||
|
renamed_formula = "{}(rec, group_by=('person',), order_by=('cat', '-Fecha'))".format(func)
|
||||||
|
|
||||||
|
# To handle "-" prefix for Date.
|
||||||
|
class Reverse(object):
|
||||||
|
def __init__(self, key):
|
||||||
|
self.key = key
|
||||||
|
def __lt__(self, other):
|
||||||
|
return other.key < self.key
|
||||||
|
|
||||||
|
self.do_test_renames(formula, renamed_formula,
|
||||||
|
calc_expected_pre = functools.partial(self.calc_expected,
|
||||||
|
group_key=(lambda r: r.Customer),
|
||||||
|
sort_key=lambda r: (SafeSortKey(r.Category), Reverse(SafeSortKey(r.Date))),
|
||||||
|
sort_reverse=(func == 'NEXT')
|
||||||
|
),
|
||||||
|
calc_expected_post = functools.partial(self.calc_expected,
|
||||||
|
group_key=(lambda r: r.person),
|
||||||
|
sort_key=lambda r: (SafeSortKey(r.cat), Reverse(SafeSortKey(r.Fecha))),
|
||||||
|
sort_reverse=(func == 'NEXT')
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_rank(self):
|
||||||
|
self.do_setup()
|
||||||
|
|
||||||
|
formula = "RANK(rec, group_by='Category', order_by='Date')"
|
||||||
|
self.add_column('Purchases', 'Rank', formula=formula)
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=[
|
||||||
|
[ "id", "Date", "Category", "Rank"],
|
||||||
|
[1, D(2023,12,1), "A", 1 ],
|
||||||
|
[2, D(2023,12,4), "A", 3 ],
|
||||||
|
[3, D(2023,12,3), "A", 2 ],
|
||||||
|
[4, D(2023,12,9), "A", 6 ],
|
||||||
|
[5, D(2023,12,2), "B", 1 ],
|
||||||
|
[6, D(2023,12,6), "B", 2 ],
|
||||||
|
[7, D(2023,12,7), "A", 5 ],
|
||||||
|
[8, D(2023,12,5), "A", 4 ],
|
||||||
|
])
|
||||||
|
formula = "RANK(rec, order_by='Date', order='desc')"
|
||||||
|
self.modify_column('Purchases', 'Rank', formula=formula)
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=[
|
||||||
|
[ "id", "Date", "Category", "Rank"],
|
||||||
|
[1, D(2023,12,1), "A", 8 ],
|
||||||
|
[2, D(2023,12,4), "A", 5 ],
|
||||||
|
[3, D(2023,12,3), "A", 6 ],
|
||||||
|
[4, D(2023,12,9), "A", 1 ],
|
||||||
|
[5, D(2023,12,2), "B", 7 ],
|
||||||
|
[6, D(2023,12,6), "B", 3 ],
|
||||||
|
[7, D(2023,12,7), "A", 2 ],
|
||||||
|
[8, D(2023,12,5), "A", 4 ],
|
||||||
|
])
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_rank_rename(self):
|
||||||
|
self.do_setup()
|
||||||
|
self.add_column('Purchases', 'Rank',
|
||||||
|
formula="RANK(rec, group_by=\"Category\", order_by='Date')")
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=[
|
||||||
|
[ "id", "Date", "Category", "Rank"],
|
||||||
|
[1, D(2023,12,1), "A", 1 ],
|
||||||
|
[2, D(2023,12,4), "A", 3 ],
|
||||||
|
[3, D(2023,12,3), "A", 2 ],
|
||||||
|
[4, D(2023,12,9), "A", 6 ],
|
||||||
|
[5, D(2023,12,2), "B", 1 ],
|
||||||
|
[6, D(2023,12,6), "B", 2 ],
|
||||||
|
[7, D(2023,12,7), "A", 5 ],
|
||||||
|
[8, D(2023,12,5), "A", 4 ],
|
||||||
|
])
|
||||||
|
|
||||||
|
self.apply_user_action(['RenameColumn', 'Purchases', 'Category', 'cat'])
|
||||||
|
self.apply_user_action(['RenameColumn', 'Purchases', 'Date', 'when'])
|
||||||
|
|
||||||
|
renamed_formula = "RANK(rec, group_by=\"cat\", order_by='when')"
|
||||||
|
self.assertTableData('_grist_Tables_column', cols="subset", rows="subset", data=[
|
||||||
|
dict(id=28, colId="Rank", formula=renamed_formula)
|
||||||
|
])
|
||||||
|
self.assertTableData('Purchases', cols="subset", data=[
|
||||||
|
[ "id", "when", "cat", "Rank"],
|
||||||
|
[1, D(2023,12,1), "A", 1 ],
|
||||||
|
[2, D(2023,12,4), "A", 3 ],
|
||||||
|
[3, D(2023,12,3), "A", 2 ],
|
||||||
|
[4, D(2023,12,9), "A", 6 ],
|
||||||
|
[5, D(2023,12,2), "B", 1 ],
|
||||||
|
[6, D(2023,12,6), "B", 2 ],
|
||||||
|
[7, D(2023,12,7), "A", 5 ],
|
||||||
|
[8, D(2023,12,5), "A", 4 ],
|
||||||
|
])
|
||||||
|
|
||||||
|
@unittest.skipUnless(six.PY3, "Python 3 only")
|
||||||
|
def test_prevnext_rename_result_attr(self):
|
||||||
|
self.do_setup()
|
||||||
|
self.add_column('Purchases', 'PrevAmount', formula="PREVIOUS(rec, order_by=None).Amount")
|
||||||
|
self.add_column('Purchases', 'NextAmount', formula="NEXT(rec, order_by=None).Amount")
|
||||||
|
self.apply_user_action(['RenameColumn', 'Purchases', 'Amount', 'Dollars'])
|
||||||
|
self.assertTableData('_grist_Tables_column', cols="subset", rows="subset", data=[
|
||||||
|
dict(id=28, colId="PrevAmount", formula="PREVIOUS(rec, order_by=None).Dollars"),
|
||||||
|
dict(id=29, colId="NextAmount", formula="NEXT(rec, order_by=None).Dollars"),
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def is_all_ints(array):
|
||||||
|
return all(isinstance(x, int) for x in array)
|
@ -0,0 +1,78 @@
|
|||||||
|
import test_engine
|
||||||
|
import testutil
|
||||||
|
from sort_key import make_sort_key
|
||||||
|
|
||||||
|
class TestSortKey(test_engine.EngineTestCase):
|
||||||
|
def test_sort_key(self):
|
||||||
|
# Set up a table with a few rows.
|
||||||
|
self.load_sample(testutil.parse_test_sample({
|
||||||
|
"SCHEMA": [
|
||||||
|
[1, "Values", [
|
||||||
|
[1, "Date", "Numeric", False, "", "", ""],
|
||||||
|
[2, "Type", "Text", False, "", "", ""],
|
||||||
|
]],
|
||||||
|
],
|
||||||
|
"DATA": {
|
||||||
|
"Values": [
|
||||||
|
["id", "Date", "Type"],
|
||||||
|
[1, 5, "a"],
|
||||||
|
[2, 4, "a"],
|
||||||
|
[3, 5, "b"],
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
|
||||||
|
table = self.engine.tables["Values"]
|
||||||
|
sort_key1 = make_sort_key(table, ("Date", "-Type"))
|
||||||
|
sort_key2 = make_sort_key(table, ("-Date", "Type"))
|
||||||
|
self.assertEqual(sorted([1, 2, 3], key=sort_key1), [2, 3, 1])
|
||||||
|
self.assertEqual(sorted([1, 2, 3], key=sort_key2), [1, 3, 2])
|
||||||
|
|
||||||
|
# Change some values
|
||||||
|
self.update_record("Values", 2, Date=6)
|
||||||
|
self.assertEqual(sorted([1, 2, 3], key=sort_key1), [3, 1, 2])
|
||||||
|
self.assertEqual(sorted([1, 2, 3], key=sort_key2), [2, 1, 3])
|
||||||
|
|
||||||
|
|
||||||
|
def test_column_rename(self):
|
||||||
|
"""
|
||||||
|
Make sure that renaming a column to another name and back does not continue using stale
|
||||||
|
references to the deleted column.
|
||||||
|
"""
|
||||||
|
# Note that SortedLookupMapColumn does retain references to the columns it uses for sorting,
|
||||||
|
# but lookup columns themselves get deleted and rebuilt in these cases (by mysterious voodoo).
|
||||||
|
|
||||||
|
# Create a simple table (People) with a couple records.
|
||||||
|
self.apply_user_action(["AddTable", "People", [
|
||||||
|
dict(id="Name", type="Text")
|
||||||
|
]])
|
||||||
|
self.add_record("People", Name="Alice")
|
||||||
|
self.add_record("People", Name="Bob")
|
||||||
|
|
||||||
|
# Create a separate table that does a lookup in the People table.
|
||||||
|
self.apply_user_action(["AddTable", "Test", [
|
||||||
|
dict(id="Lookup1", type="Any", isFormula=True,
|
||||||
|
formula="People.lookupOne(order_by='-Name').Name"),
|
||||||
|
dict(id="Lookup2", type="Any", isFormula=True,
|
||||||
|
formula="People.lookupOne(order_by='Name').Name"),
|
||||||
|
dict(id="Lookup3", type="Any", isFormula=True,
|
||||||
|
formula="People.lookupOne(Name='Bob').Name"),
|
||||||
|
]])
|
||||||
|
self.add_record("Test")
|
||||||
|
|
||||||
|
# Test that lookups return data as expected.
|
||||||
|
self.assertTableData('Test', cols="subset", data=[
|
||||||
|
dict(id=1, Lookup1="Bob", Lookup2="Alice", Lookup3="Bob")
|
||||||
|
])
|
||||||
|
|
||||||
|
# Rename a column used for lookups or order_by. Lookup result shouldn't change.
|
||||||
|
self.apply_user_action(["RenameColumn", "People", "Name", "FullName"])
|
||||||
|
self.assertTableData('Test', cols="subset", data=[
|
||||||
|
dict(id=1, Lookup1="Bob", Lookup2="Alice", Lookup3="Bob")
|
||||||
|
])
|
||||||
|
|
||||||
|
# Rename the column back. Lookup result shouldn't change.
|
||||||
|
self.apply_user_action(["RenameColumn", "People", "FullName", "Name"])
|
||||||
|
self.assertTableData('Test', cols="subset", data=[
|
||||||
|
dict(id=1, Lookup1="Bob", Lookup2="Alice", Lookup3="Bob")
|
||||||
|
])
|
Loading…
Reference in new issue