Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) Nice summary table IDs

Browse Source 
Summary:
Changes auto-generated summary table IDs from e.g. `GristSummary_6_Table1` to `Table1_summary_A_B` (meaning `Table1` grouped by `A` and `B`). This makes it easier to write formulas involving summary tables, make API requests, understand logs, etc.

Because these don't encode the source table ID as reliably as before, `decode_summary_table_name` now uses the summary table schema info, not just the summary table ID. Specifically, it looks at the type of the `group` column, which is `RefList:<source table id>`.

Renaming a source table renames the summary table as before, and now renaming a groupby column renames the summary table as well.

Conflicting table names are resolved in the usual way by adding a number at the end, e.g. `Table1_summary_A_B2`. These summary tables are not automatically renamed when the disambiguation is no longer needed.

A new migration renames all summary tables to the new scheme, and updates formulas using summary tables with a simple regex.

Test Plan:
Updated many tests to use the new style of name.

Added new Python tests to for resolving conflicts when renaming source tables and groupby columns.

Added a test for the migration, including renames in formulas.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3508
This commit is contained in:
Alex Hall
2022-07-11 20:00:25 +02:00
parent f1df6c0a46
commit b8486dcdba
18 changed files with 507 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
sandbox/grist/test_summary.py
View File

@@ -66,25 +66,10 @@ class TestSummary(test_engine.EngineTestCase):
#----------------------------------------------------------------------
def test_encode_summary_table_name(self):
self.assertEqual(summary.encode_summary_table_name("Foo"), "GristSummary_3_Foo")
self.assertEqual(summary.encode_summary_table_name("Foo2"), "GristSummary_4_Foo2")
self.assertEqual(summary.decode_summary_table_name("GristSummary_3_Foo"), "Foo")
self.assertEqual(summary.decode_summary_table_name("GristSummary_4_Foo2"), "Foo2")
self.assertEqual(summary.decode_summary_table_name("GristSummary_3_Foo2"), "Foo")
self.assertEqual(summary.decode_summary_table_name("GristSummary_4_Foo2_2"), "Foo2")
# Test that underscore in the name is OK.
self.assertEqual(summary.decode_summary_table_name("GristSummary_5_Foo_234"), "Foo_2")
self.assertEqual(summary.decode_summary_table_name("GristSummary_4_Foo_234"), "Foo_")
self.assertEqual(summary.decode_summary_table_name("GristSummary_6__Foo_234"), "_Foo_2")
# Test that we return None for invalid values.
self.assertEqual(summary.decode_summary_table_name("Foo2"), None)
self.assertEqual(summary.decode_summary_table_name("GristSummary_3Foo"), None)
self.assertEqual(summary.decode_summary_table_name("GristSummary_4_Foo"), None)
self.assertEqual(summary.decode_summary_table_name("GristSummary_3X_Foo"), None)
self.assertEqual(summary.decode_summary_table_name("_5_Foo_234"), None)
self.assertEqual(summary.decode_summary_table_name("_GristSummary_3_Foo"), None)
self.assertEqual(summary.decode_summary_table_name("gristsummary_3_Foo"), None)
self.assertEqual(summary.decode_summary_table_name("GristSummary3_Foo"), None)
self.assertEqual(summary.encode_summary_table_name("Foo", []), "Foo_summary")
self.assertEqual(summary.encode_summary_table_name("Foo", ["A"]), "Foo_summary_A")
self.assertEqual(summary.encode_summary_table_name("Foo", ["A", "B"]), "Foo_summary_A_B")
self.assertEqual(summary.encode_summary_table_name("Foo", ["B", "A"]), "Foo_summary_A_B")
#----------------------------------------------------------------------
@@ -116,7 +101,7 @@ class TestSummary(test_engine.EngineTestCase):
# Verify that a new table gets created, and a new view, with a section for that table,
# and some auto-generated summary fields.
summary_table1 = Table(2, "GristSummary_7_Address", primaryViewId=0, summarySourceTable=1,
summary_table1 = Table(2, "Address_summary", primaryViewId=0, summarySourceTable=1,
columns=[
Column(14, "group", "RefList:Address", isFormula=True, summarySourceCol=0,
formula="table.getSummarySourceGroup(rec)"),
@@ -135,7 +120,7 @@ class TestSummary(test_engine.EngineTestCase):
self.assertViews([basic_view, summary_view1])
# Verify the summarized data.
self.assertTableData('GristSummary_7_Address', cols="subset", data=[
self.assertTableData('Address_summary', cols="subset", data=[
[ "id", "count", "amount"],
[ 1, 11, 66.0 ],
])
@@ -145,7 +130,7 @@ class TestSummary(test_engine.EngineTestCase):
# Verify that a new table gets created again, a new view, and a section for that table.
# Note that we also check that summarySourceTable and summarySourceCol fields are correct.
summary_table2 = Table(3, "GristSummary_7_Address2", primaryViewId=0, summarySourceTable=1,
summary_table2 = Table(3, "Address_summary_state", primaryViewId=0, summarySourceTable=1,
columns=[
Column(17, "state", "Text", isFormula=False, formula="", summarySourceCol=12),
Column(18, "group", "RefList:Address", isFormula=True, summarySourceCol=0,
@@ -173,7 +158,7 @@ class TestSummary(test_engine.EngineTestCase):
])
# Verify the summarized data.
self.assertTableData('GristSummary_7_Address2', cols="subset", data=[
self.assertTableData('Address_summary_state', cols="subset", data=[
[ "id", "state", "count", "amount" ],
[ 1, "NY", 7, 1.+2+6+7+8+10+11 ],
[ 2, "WA", 1, 3. ],
@@ -185,7 +170,7 @@ class TestSummary(test_engine.EngineTestCase):
self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])
# Verify the new table and views.
summary_table3 = Table(4, "GristSummary_7_Address3", primaryViewId=0, summarySourceTable=1,
summary_table3 = Table(4, "Address_summary_city_state", primaryViewId=0, summarySourceTable=1,
columns=[
Column(21, "city", "Text", isFormula=False, formula="", summarySourceCol=11),
Column(22, "state", "Text", isFormula=False, formula="", summarySourceCol=12),
@@ -208,7 +193,7 @@ class TestSummary(test_engine.EngineTestCase):
self.assertViews([basic_view, summary_view1, summary_view2, summary_view3])
# Verify the summarized data.
self.assertTableData('GristSummary_7_Address3', cols="subset", data=[
self.assertTableData('Address_summary_city_state', cols="subset", data=[
[ "id", "city", "state", "count", "amount" ],
[ 1, "New York", "NY" , 3, 1.+6+11 ],
[ 2, "Albany", "NY" , 1, 2. ],
@@ -269,7 +254,7 @@ class Address:
self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])
# Verify the new table and views.
summary_table = Table(2, "GristSummary_7_Address", primaryViewId=0, summarySourceTable=1,
summary_table = Table(2, "Address_summary_city_state", primaryViewId=0, summarySourceTable=1,
columns=[
Column(14, "city", "Text", isFormula=False, formula="", summarySourceCol=11),
Column(15, "state", "Text", isFormula=False, formula="", summarySourceCol=12),
@@ -316,7 +301,7 @@ class Address:
self.assertViews([summary_view, summary_view2, summary_view3])
# Verify the summarized data.
self.assertTableData('GristSummary_7_Address', cols="subset", data=[
self.assertTableData('Address_summary_city_state', cols="subset", data=[
[ "id", "city", "state", "count", "amount" ],
[ 1, "New York", "NY" , 3, 1.+6+11 ],
[ 2, "Albany", "NY" , 1, 2. ],
@@ -342,12 +327,12 @@ class Address:
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary", 0, 1, columns=[
Column(14, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(15, "count", "Int", True, "len($group)", 0),
Column(16, "amount", "Numeric", True, "SUM($group.amount)", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary_state", 0, 1, columns=[
Column(17, "state", "Text", False, "", 12),
Column(18, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(19, "count", "Int", True, "len($group)", 0),
@@ -370,8 +355,8 @@ class Address:
self.assertTableData("_grist_Tables", cols="subset", data=[
['id', 'tableId', 'summarySourceTable'],
[ 1, 'Address', 0],
[ 2, 'GristSummary_7_Address', 1],
[ 3, 'GristSummary_7_Address2', 1],
[ 2, 'Address_summary', 1],
[ 3, 'Address_summary_state', 1],
[ 4, 'Address2', 0],
])
@@ -382,12 +367,12 @@ class Address:
# Make sure this creates new section rather than reuses similar ones for the wrong table.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary", 0, 1, columns=[
Column(14, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(15, "count", "Int", True, "len($group)", 0),
Column(16, "amount", "Numeric", True, "SUM($group.amount)", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary_state", 0, 1, columns=[
Column(17, "state", "Text", False, "", 12),
Column(18, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(19, "count", "Int", True, "len($group)", 0),
@@ -399,12 +384,12 @@ class Address:
Column(23, "state", "Text", False, "", 0),
Column(24, "amount", "Numeric", False, "", 0),
]),
Table(5, "GristSummary_8_Address2", 0, 4, columns=[
Table(5, "Address2_summary", 0, 4, columns=[
Column(25, "group", "RefList:Address2", True, "table.getSummarySourceGroup(rec)", 0),
Column(26, "count", "Int", True, "len($group)", 0),
Column(27, "amount", "Numeric", True, "SUM($group.amount)", 0),
]),
Table(6, "GristSummary_8_Address2_2", 0, 4, columns=[
Table(6, "Address2_summary_state", 0, 4, columns=[
Column(28, "state", "Text", False, "", 23),
Column(29, "group", "RefList:Address2", True, "table.getSummarySourceGroup(rec)", 0),
Column(30, "count", "Int", True, "len($group)", 0),
@@ -424,7 +409,7 @@ class Address:
self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])
# Verify that the summary table respects all updates to the source table.
self._do_test_updates("Address", "GristSummary_7_Address")
self._do_test_updates("Address", "Address_summary_city_state")
def _do_test_updates(self, source_tbl_name, summary_tbl_name):
# This is the main part of test_summary_updates(). It's moved to its own method so that
@@ -541,7 +526,7 @@ class Address:
# Check what tables we have now.
self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
[1, "Address", 0],
[2, "GristSummary_7_Address", 1],
[2, "Address_summary_city_state", 1],
])
# Rename the table: this is what we are really testing in this test case.
@@ -549,11 +534,11 @@ class Address:
self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
[1, "Location", 0],
[2, "GristSummary_8_Location", 1],
[2, "Location_summary_city_state", 1],
])
# Verify that the bigger summary table respects all updates to the renamed source table.
self._do_test_updates("Location", "GristSummary_8_Location")
self._do_test_updates("Location", "Location_summary_city_state")
#----------------------------------------------------------------------
@@ -565,11 +550,11 @@ class Address:
self.apply_user_action(["CreateViewSection", 1, 0, "record", [], None])
self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
[1, "Address", 0],
[2, "GristSummary_7_Address", 1],
[3, "GristSummary_7_Address2", 1],
[2, "Address_summary_city_state", 1],
[3, "Address_summary", 1],
])
# Verify the data in the simple totals-only summary table.
self.assertTableData('GristSummary_7_Address2', cols="subset", data=[
self.assertTableData('Address_summary', cols="subset", data=[
[ "id", "count", "amount"],
[ 1, 11, 66.0 ],
])
@@ -578,21 +563,21 @@ class Address:
self.apply_user_action(["RenameTable", "Address", "Addresses"])
self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
[1, "Addresses", 0],
[2, "GristSummary_9_Addresses", 1],
[3, "GristSummary_9_Addresses2", 1],
[2, "Addresses_summary_city_state", 1],
[3, "Addresses_summary", 1],
])
self.assertTableData('GristSummary_9_Addresses2', cols="subset", data=[
self.assertTableData('Addresses_summary', cols="subset", data=[
[ "id", "count", "amount"],
[ 1, 11, 66.0 ],
])
# Remove one of the tables so that we can use _do_test_updates to verify updates still work.
self.apply_user_action(["RemoveTable", "GristSummary_9_Addresses2"])
self.apply_user_action(["RemoveTable", "Addresses_summary"])
self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
[1, "Addresses", 0],
[2, "GristSummary_9_Addresses", 1],
[2, "Addresses_summary_city_state", 1],
])
self._do_test_updates("Addresses", "GristSummary_9_Addresses")
self._do_test_updates("Addresses", "Addresses_summary_city_state")
#----------------------------------------------------------------------
@@ -610,14 +595,14 @@ class Address:
# These are the tables and columns we automatically get.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_state", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "state", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(17, "count", "Int", True, "len($group)", 0),
Column(18, "amount", "Numeric", True, "SUM($group.amount)", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary", 0, 1, columns=[
Column(19, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "count", "Int", True, "len($group)", 0),
Column(21, "amount", "Numeric", True, "SUM($group.amount)", 0),
@@ -626,7 +611,7 @@ class Address:
# Now change a formula using one of the summary tables. It should trigger an equivalent
# change in the other.
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "amount",
self.apply_user_action(["ModifyColumn", "Address_summary_city_state", "amount",
{"formula": "10*sum($group.amount)"}])
self.assertTableData('_grist_Tables_column', rows="subset", cols="subset", data=[
['id', 'colId', 'type', 'formula', 'widgetOptions', 'label'],
@@ -635,7 +620,7 @@ class Address:
])
# Change a formula and a few other fields in the other table, and verify a change to both.
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address2", "amount",
self.apply_user_action(["ModifyColumn", "Address_summary", "amount",
{"formula": "100*sum($group.amount)",
"type": "Text",
"widgetOptions": "hello",
@@ -649,7 +634,7 @@ class Address:
])
# Check the values in the summary tables: they should reflect the new formula.
self.assertTableData('GristSummary_7_Address', cols="subset", data=[
self.assertTableData('Address_summary_city_state', cols="subset", data=[
[ "id", "city", "state", "count", "amount" ],
[ 1, "New York", "NY" , 3, str(100*(1+6+11))],
[ 2, "Albany", "NY" , 1, "200" ],
@@ -661,7 +646,7 @@ class Address:
[ 8, "Boston", "MA" , 1, "900" ],
[ 9, "Yonkers", "NY" , 1, "1000" ],
])
self.assertTableData('GristSummary_7_Address2', cols="subset", data=[
self.assertTableData('Address_summary', cols="subset", data=[
[ "id", "count", "amount"],
[ 1, 11, "6600"],
])
@@ -670,19 +655,19 @@ class Address:
self.apply_user_action(["CreateViewSection", 1, 0, "record", [12], None])
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_state", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "state", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(17, "count", "Int", True, "len($group)", 0),
Column(18, "amount", "Text", True, "100*sum($group.amount)", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary", 0, 1, columns=[
Column(19, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "count", "Int", True, "len($group)", 0),
Column(21, "amount", "Text", True, "100*sum($group.amount)", 0),
]),
Table(4, "GristSummary_7_Address3", 0, 1, columns=[
Table(4, "Address_summary_state", 0, 1, columns=[
Column(22, "state", "Text", False, "", 12),
Column(23, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(24, "count", "Int", True, "len($group)", 0),
@@ -697,7 +682,7 @@ class Address:
])
# Verify the summarized data.
self.assertTableData('GristSummary_7_Address3', cols="subset", data=[
self.assertTableData('Address_summary_state', cols="subset", data=[
[ "id", "state", "count", "amount" ],
[ 1, "NY", 7, str(int(100*(1.+2+6+7+8+10+11))) ],
[ 2, "WA", 1, "300" ],
@@ -722,7 +707,7 @@ class Address:
Column(3, "B", "Numeric", False, "", 0),
Column(4, "C", "Any", True, "", 0),
]),
Table(2, "GristSummary_6_Table1", summarySourceTable=1, primaryViewId=0, columns=[
Table(2, "Table1_summary_A", summarySourceTable=1, primaryViewId=0, columns=[
Column(5, "A", "Numeric", False, "", 2),
Column(6, "group", "RefList:Table1", True, "table.getSummarySourceGroup(rec)", 0),
Column(7, "count", "Int", True, "len($group)", 0),
@@ -735,7 +720,7 @@ class Address:
[ 2, 2.0, 20, 2.0, None ],
[ 3, 3.0, 10, 3.0, None ],
])
self.assertTableData('GristSummary_6_Table1', data=[
self.assertTableData('Table1_summary_A', data=[
[ "id", "A", "group", "count", "B" ],
[ 1, 10, [1,3], 2, 4 ],
[ 2, 20, [2], 1, 2 ],
@@ -753,7 +738,7 @@ class Address:
Column(3, "B", "Numeric", False, "", 0),
Column(4, "C", "Any", True, "", 0),
]),
Table(2, "GristSummary_6_Table1", summarySourceTable=1, primaryViewId=0, columns=[
Table(2, "Table1_summary_A", summarySourceTable=1, primaryViewId=0, columns=[
Column(5, "A", "Text", False, "", 2),
Column(6, "group", "RefList:Table1", True, "table.getSummarySourceGroup(rec)", 0),
Column(7, "count", "Int", True, "len($group)", 0),
@@ -766,7 +751,7 @@ class Address:
[ 2, 2.0, "20", 2.0, None ],
[ 3, 3.0, "10", 3.0, None ],
])
self.assertTableData('GristSummary_6_Table1', data=[
self.assertTableData('Table1_summary_A', data=[
[ "id", "A", "group", "count", "B" ],
[ 1, "10", [1,3], 2, 4 ],
[ 2, "20", [2], 1, 2 ],
@@ -791,7 +776,7 @@ class Address:
Column(3, "B", "Numeric", False, "", 0),
Column(4, "C", "Numeric", False, "", 0),
]),
Table(2, "GristSummary_6_Table1", summarySourceTable=1, primaryViewId=0, columns=[
Table(2, "Table1_summary_A_B", summarySourceTable=1, primaryViewId=0, columns=[
Column(5, "A", "Text", False, "", 2),
Column(6, "B", "Numeric", False, "", 3),
Column(7, "group", "RefList:Table1", True, "table.getSummarySourceGroup(rec)", 0),
@@ -805,7 +790,7 @@ class Address:
[ 2, 2.0, 'b', 1.0, 5 ],
[ 3, 3.0, 'c', 2.0, 6 ],
])
self.assertTableData('GristSummary_6_Table1', data=[
self.assertTableData('Table1_summary_A_B', data=[
[ "id", "A", "B", "group", "count", "C" ],
[ 1, 'a', 1.0, [1], 1, 4 ],
[ 2, 'b', 1.0, [2], 1, 5 ],
@@ -822,7 +807,7 @@ class Address:
Column(3, "B", "Numeric", False, "", 0),
Column(4, "C", "Numeric", False, "", 0),
]),
Table(3, "GristSummary_6_Table1_2", summarySourceTable=1, primaryViewId=0, columns=[
Table(3, "Table1_summary_B", summarySourceTable=1, primaryViewId=0, columns=[
Column(10, "B", "Numeric", False, "", 3),
Column(12, "count", "Int", True, "len($group)", 0),
Column(13, "C", "Numeric", True, "SUM($group.C)", 0),
@@ -835,7 +820,7 @@ class Address:
[ 2, 2.0, 1.0, 5 ],
[ 3, 3.0, 2.0, 6 ],
])
self.assertTableData('GristSummary_6_Table1_2', data=[
self.assertTableData('Table1_summary_B', data=[
[ "id", "B", "group", "count", "C" ],
[ 1, 1.0, [1,2], 2, 9 ],
[ 2, 2.0, [3], 1, 6 ],