mirror of
https://github.com/gristlabs/grist-core.git
synced 2026-03-02 04:09:24 +00:00
(core) move data engine code to core
Summary: this moves sandbox/grist to core, and adds a requirements.txt file for reconstructing the content of sandbox/thirdparty. Test Plan: existing tests pass. Tested core functionality manually. Tested docker build manually. Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2563
This commit is contained in:
Paul Fitzpatrick
294
sandbox/grist/test_derived.py
Normal file
294
sandbox/grist/test_derived.py
Normal file
@@ -0,0 +1,294 @@
|
||||
import actions
|
||||
import logger
|
||||
|
||||
import testutil
|
||||
import test_engine
|
||||
|
||||
log = logger.Logger(__name__, logger.INFO)
|
||||
|
||||
def _bulk_update(table_name, col_names, row_data):
|
||||
return actions.BulkUpdateRecord(
|
||||
*testutil.table_data_from_rows(table_name, col_names, row_data))
|
||||
|
||||
class TestDerived(test_engine.EngineTestCase):
|
||||
sample = testutil.parse_test_sample({
|
||||
"SCHEMA": [
|
||||
[1, "Customers", [
|
||||
[1, "firstName", "Text", False, "", "", ""],
|
||||
[2, "lastName", "Text", False, "", "", ""],
|
||||
[3, "state", "Text", False, "", "", ""],
|
||||
]],
|
||||
[2, "Orders", [
|
||||
[10, "year", "Int", False, "", "", ""],
|
||||
[11, "customer", "Ref:Customers", False, "", "", ""],
|
||||
[12, "product", "Text", False, "", "", ""],
|
||||
[13, "amount", "Numeric", False, "", "", ""],
|
||||
]],
|
||||
],
|
||||
"DATA": {
|
||||
"Customers": [
|
||||
["id", "firstName", "lastName", "state"],
|
||||
[1, "Lois", "Long", "NY"],
|
||||
[2, "Felix", "Myers", "NY"],
|
||||
[3, "Grace", "Hawkins", "CT"],
|
||||
[4, "Bessie", "Green", "NJ"],
|
||||
[5, "Jerome", "Daniel", "CT"],
|
||||
],
|
||||
"Orders": [
|
||||
["id", "year", "customer", "product", "amount" ],
|
||||
[1, 2012, 3, "A", 15 ],
|
||||
[2, 2013, 2, "A", 15 ],
|
||||
[3, 2013, 3, "A", 15 ],
|
||||
[4, 2014, 1, "B", 35 ],
|
||||
[5, 2014, 5, "B", 35 ],
|
||||
[6, 2014, 3, "A", 16 ],
|
||||
[7, 2015, 1, "A", 17 ],
|
||||
[8, 2015, 2, "B", 36 ],
|
||||
[9, 2015, 3, "B", 36 ],
|
||||
[10, 2015, 5, "A", 17 ],
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
def test_group_by_one(self):
|
||||
"""
|
||||
Test basic summary table operation, for a table grouped by one columns.
|
||||
"""
|
||||
self.load_sample(self.sample)
|
||||
|
||||
# Create a derived table summarizing count and total of orders by year.
|
||||
self.apply_user_action(["CreateViewSection", 2, 0, 'record', [10]])
|
||||
|
||||
# Check the results.
|
||||
self.assertPartialData("GristSummary_6_Orders", ["id", "year", "count", "amount", "group" ], [
|
||||
[1, 2012, 1, 15, [1]],
|
||||
[2, 2013, 2, 30, [2,3]],
|
||||
[3, 2014, 3, 86, [4,5,6]],
|
||||
[4, 2015, 4, 106, [7,8,9,10]],
|
||||
])
|
||||
|
||||
# Updating amounts should cause totals to be updated in the summary.
|
||||
out_actions = self.update_records("Orders", ["id", "amount"], [
|
||||
[1, 14],
|
||||
[2, 14]
|
||||
])
|
||||
self.assertPartialOutActions(out_actions, {
|
||||
"calc": [actions.BulkUpdateRecord("GristSummary_6_Orders", [1,2], {'amount': [14, 29]})],
|
||||
"calls": {"GristSummary_6_Orders": {"amount": 2}}
|
||||
})
|
||||
|
||||
# Changing a record from one product to another should cause the two affected lines to change.
|
||||
out_actions = self.update_record("Orders", 10, year=2012)
|
||||
self.assertPartialOutActions(out_actions, {
|
||||
"stored": [actions.UpdateRecord("Orders", 10, {"year": 2012})],
|
||||
"calc": [
|
||||
actions.BulkUpdateRecord("GristSummary_6_Orders", [1,4], {"group": [[1,10], [7,8,9]]}),
|
||||
actions.BulkUpdateRecord("GristSummary_6_Orders", [1,4], {"amount": [31.0, 89.0]}),
|
||||
actions.BulkUpdateRecord("GristSummary_6_Orders", [1,4], {"count": [2,3]}),
|
||||
],
|
||||
"calls": {"GristSummary_6_Orders": {"group": 2, "amount": 2, "count": 2},
|
||||
"Orders": {"#lookup##summary#GristSummary_6_Orders": 1,
|
||||
"#summary#GristSummary_6_Orders": 1}}
|
||||
})
|
||||
|
||||
self.assertPartialData("GristSummary_6_Orders", ["id", "year", "count", "amount", "group" ], [
|
||||
[1, 2012, 2, 31.0, [1,10]],
|
||||
[2, 2013, 2, 29.0, [2,3]],
|
||||
[3, 2014, 3, 86.0, [4,5,6]],
|
||||
[4, 2015, 3, 89.0, [7,8,9]],
|
||||
])
|
||||
|
||||
# Changing a record to a new year that wasn't in the summary should cause an add-record.
|
||||
out_actions = self.update_record("Orders", 10, year=1999)
|
||||
self.assertPartialOutActions(out_actions, {
|
||||
"stored": [
|
||||
actions.UpdateRecord("Orders", 10, {"year": 1999}),
|
||||
actions.AddRecord("GristSummary_6_Orders", 5, {'year': 1999}),
|
||||
],
|
||||
"calc": [
|
||||
actions.BulkUpdateRecord("GristSummary_6_Orders", [1,5], {"group": [[1], [10]]}),
|
||||
actions.BulkUpdateRecord("GristSummary_6_Orders", [1,5], {"amount": [14.0, 17.0]}),
|
||||
actions.BulkUpdateRecord("GristSummary_6_Orders", [1,5], {"count": [1,1]}),
|
||||
],
|
||||
"calls": {
|
||||
"GristSummary_6_Orders": {'#lookup#year': 1, "group": 2, "amount": 2, "count": 2},
|
||||
"Orders": {"#lookup##summary#GristSummary_6_Orders": 2,
|
||||
"#summary#GristSummary_6_Orders": 2}}
|
||||
})
|
||||
|
||||
self.assertPartialData("GristSummary_6_Orders", ["id", "year", "count", "amount", "group" ], [
|
||||
[1, 2012, 1, 14.0, [1]],
|
||||
[2, 2013, 2, 29.0, [2,3]],
|
||||
[3, 2014, 3, 86.0, [4,5,6]],
|
||||
[4, 2015, 3, 89.0, [7,8,9]],
|
||||
[5, 1999, 1, 17.0, [10]],
|
||||
])
|
||||
|
||||
|
||||
def test_group_by_two(self):
|
||||
"""
|
||||
Test a summary table created by grouping on two columns.
|
||||
"""
|
||||
self.load_sample(self.sample)
|
||||
|
||||
self.apply_user_action(["CreateViewSection", 2, 0, 'record', [10, 12]])
|
||||
self.assertPartialData("GristSummary_6_Orders", [
|
||||
"id", "year", "product", "count", "amount", "group"
|
||||
], [
|
||||
[1, 2012, "A", 1, 15.0, [1]],
|
||||
[2, 2013, "A", 2, 30.0, [2,3]],
|
||||
[3, 2014, "B", 2, 70.0, [4,5]],
|
||||
[4, 2014, "A", 1, 16.0, [6]],
|
||||
[5, 2015, "A", 2, 34.0, [7,10]],
|
||||
[6, 2015, "B", 2, 72.0, [8,9]],
|
||||
])
|
||||
|
||||
# Changing a record from one product to another should cause the two affected lines to change,
|
||||
# or new lines to be created as needed.
|
||||
out_actions = self.update_records("Orders", ["id", "product"], [
|
||||
[2, "B"],
|
||||
[6, "B"],
|
||||
[7, "C"],
|
||||
])
|
||||
self.assertPartialOutActions(out_actions, {
|
||||
"stored": [
|
||||
actions.BulkUpdateRecord("Orders", [2, 6, 7], {"product": ["B", "B", "C"]}),
|
||||
actions.AddRecord("GristSummary_6_Orders", 7, {'year': 2013, 'product': 'B'}),
|
||||
actions.AddRecord("GristSummary_6_Orders", 8, {'year': 2015, 'product': 'C'}),
|
||||
],
|
||||
"calc": [
|
||||
actions.BulkUpdateRecord("GristSummary_6_Orders", [2,3,4,5,7,8], {
|
||||
"group": [[3], [4,5,6], [], [10], [2], [7]]
|
||||
}),
|
||||
actions.BulkUpdateRecord("GristSummary_6_Orders", [2,3,4,5,7,8], {
|
||||
"amount": [15.0, 86.0, 0, 17.0, 15.0, 17.0]
|
||||
}),
|
||||
actions.BulkUpdateRecord("GristSummary_6_Orders", [2,3,4,5,7,8], {
|
||||
"count": [1, 3, 0, 1, 1, 1]
|
||||
}),
|
||||
],
|
||||
})
|
||||
|
||||
# Verify the results.
|
||||
self.assertPartialData("GristSummary_6_Orders", [
|
||||
"id", "year", "product", "count", "amount", "group"
|
||||
], [
|
||||
[1, 2012, "A", 1, 15.0, [1]],
|
||||
[2, 2013, "A", 1, 15.0, [3]],
|
||||
[3, 2014, "B", 3, 86.0, [4,5,6]],
|
||||
[4, 2014, "A", 0, 0.0, []],
|
||||
[5, 2015, "A", 1, 17.0, [10]],
|
||||
[6, 2015, "B", 2, 72.0, [8,9]],
|
||||
[7, 2013, "B", 1, 15.0, [2]],
|
||||
[8, 2015, "C", 1, 17.0, [7]],
|
||||
])
|
||||
|
||||
def test_group_with_references(self):
|
||||
"""
|
||||
Test summary tables grouped on indirect values. In this example we want for each
|
||||
customer.state, the number of customers and the total of their orders, which we can do either
|
||||
as a summary on the Customers table, or a summary on the Orders table.
|
||||
"""
|
||||
self.load_sample(self.sample)
|
||||
|
||||
# Create a summary on the Customers table. Adding orders involves a lookup for each customer.
|
||||
self.apply_user_action(["CreateViewSection", 1, 0, 'record', [3]])
|
||||
self.add_column("GristSummary_9_Customers", "totalAmount",
|
||||
formula="sum(sum(Orders.lookupRecords(customer=c).amount) for c in $group)")
|
||||
|
||||
self.assertPartialData("GristSummary_9_Customers", ["id", "state", "count", "totalAmount"], [
|
||||
[1, "NY", 2, 103.0 ],
|
||||
[2, "CT", 2, 134.0 ],
|
||||
[3, "NJ", 1, 0.0 ],
|
||||
])
|
||||
|
||||
# # Create the same summary on the Orders table, looking up 'state' via the Customer reference.
|
||||
# self.apply_user_action(["AddDerivedTableSource", "Summary4", "Orders",
|
||||
# {"state": "$customer.state"}])
|
||||
# self.add_column("Summary4", "numCustomers", formula="len(set($source_Orders.customer))")
|
||||
# self.add_column("Summary4", "totalAmount", formula="sum($source_Orders.amount)")
|
||||
|
||||
# self.assertPartialData("Summary4", ["id", "state", "numCustomers", "totalAmount"], [
|
||||
# [1, "CT", 2, 134.0 ],
|
||||
# [2, "NY", 2, 103.0 ],
|
||||
# ])
|
||||
|
||||
# In either case, changing an amount (from 36->37 for a CT customer) should update summaries.
|
||||
out_actions = self.update_record('Orders', 9, amount=37)
|
||||
self.assertPartialOutActions(out_actions, {
|
||||
"calc": [actions.UpdateRecord("GristSummary_9_Customers", 2, {"totalAmount": 135.0})]
|
||||
})
|
||||
|
||||
# In either case, changing a customer's state should trigger recomputation too.
|
||||
# We are changing a NY customer with $51 in orders to MA.
|
||||
self.update_record('Customers', 2, state="MA")
|
||||
self.assertPartialData("GristSummary_9_Customers", ["id", "state", "count", "totalAmount"], [
|
||||
[1, "NY", 1, 52.0 ],
|
||||
[2, "CT", 2, 135.0 ],
|
||||
[3, "NJ", 1, 0.0 ],
|
||||
[4, "MA", 1, 51.0 ],
|
||||
])
|
||||
# self.assertPartialData("Summary4", ["id", "state", "numCustomers", "totalAmount"], [
|
||||
# [1, "CT", 2, 135.0 ],
|
||||
# [2, "NY", 1, 52.0 ],
|
||||
# [3, "MA", 1, 51.0 ],
|
||||
# ])
|
||||
|
||||
# Similarly, changing an Order to refer to a different customer should update both tables.
|
||||
# Here we are changing a $17 order (#7) for a NY customer (#1) to a NJ customer (#4).
|
||||
out_actions = self.update_record("Orders", 7, customer=4)
|
||||
# self.assertPartialOutActions(out_actions, {
|
||||
# "stored": [actions.UpdateRecord("Orders", 7, {"customer": 4}),
|
||||
# actions.AddRecord("Summary4", 4, {"state": "NJ"}),
|
||||
# actions.UpdateRecord("Summary4", 4, {"manualSort": 4.0})]
|
||||
# })
|
||||
self.assertPartialData("GristSummary_9_Customers", ["id", "state", "count", "totalAmount"], [
|
||||
[1, "NY", 1, 35.0 ],
|
||||
[2, "CT", 2, 135.0 ],
|
||||
[3, "NJ", 1, 17.0 ],
|
||||
[4, "MA", 1, 51.0 ],
|
||||
])
|
||||
# self.assertPartialData("Summary4", ["id", "state", "numCustomers", "totalAmount"], [
|
||||
# [1, "CT", 2, 135.0 ],
|
||||
# [2, "NY", 1, 35.0 ],
|
||||
# [3, "MA", 1, 51.0 ],
|
||||
# [4, "NJ", 1, 17.0 ],
|
||||
# ])
|
||||
|
||||
def test_deletions(self):
|
||||
self.load_sample(self.sample)
|
||||
|
||||
# Create a summary table summarizing count and total of orders by year.
|
||||
self.apply_user_action(["CreateViewSection", 2, 0, 'record', [10]])
|
||||
self.assertPartialData("GristSummary_6_Orders", ["id", "year", "count", "amount", "group" ], [
|
||||
[1, 2012, 1, 15.0, [1]],
|
||||
[2, 2013, 2, 30.0, [2,3]],
|
||||
[3, 2014, 3, 86.0, [4,5,6]],
|
||||
[4, 2015, 4, 106.0, [7,8,9,10]],
|
||||
])
|
||||
|
||||
# Update a record so that a new line appears in the summary table.
|
||||
out_actions_update = self.update_record("Orders", 1, year=2007)
|
||||
self.assertPartialData("GristSummary_6_Orders", ["id", "year", "count", "amount", "group" ], [
|
||||
[1, 2012, 0, 0.0, []],
|
||||
[2, 2013, 2, 30.0, [2,3]],
|
||||
[3, 2014, 3, 86.0, [4,5,6]],
|
||||
[4, 2015, 4, 106.0, [7,8,9,10]],
|
||||
[5, 2007, 1, 15.0, [1]],
|
||||
])
|
||||
|
||||
# Undo and ensure that the new line is gone from the summary table.
|
||||
out_actions_undo = self.apply_undo_actions(out_actions_update.undo)
|
||||
self.assertPartialData("GristSummary_6_Orders", ["id", "year", "count", "amount", "group" ], [
|
||||
[1, 2012, 1, 15.0, [1]],
|
||||
[2, 2013, 2, 30.0, [2,3]],
|
||||
[3, 2014, 3, 86.0, [4,5,6]],
|
||||
[4, 2015, 4, 106.0, [7,8,9,10]],
|
||||
])
|
||||
self.assertPartialOutActions(out_actions_undo, {
|
||||
"stored": [actions.RemoveRecord("GristSummary_6_Orders", 5),
|
||||
actions.UpdateRecord("Orders", 1, {"year": 2012})],
|
||||
"calls": {"GristSummary_6_Orders": {"group": 1, "amount": 1, "count": 1},
|
||||
"Orders": {"#lookup##summary#GristSummary_6_Orders": 2,
|
||||
"#summary#GristSummary_6_Orders": 2}}
|
||||
})
|
||||
Reference in New Issue
Block a user