Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) Nice summary table IDs

Browse Source 
Summary:
Changes auto-generated summary table IDs from e.g. `GristSummary_6_Table1` to `Table1_summary_A_B` (meaning `Table1` grouped by `A` and `B`). This makes it easier to write formulas involving summary tables, make API requests, understand logs, etc.

Because these don't encode the source table ID as reliably as before, `decode_summary_table_name` now uses the summary table schema info, not just the summary table ID. Specifically, it looks at the type of the `group` column, which is `RefList:<source table id>`.

Renaming a source table renames the summary table as before, and now renaming a groupby column renames the summary table as well.

Conflicting table names are resolved in the usual way by adding a number at the end, e.g. `Table1_summary_A_B2`. These summary tables are not automatically renamed when the disambiguation is no longer needed.

A new migration renames all summary tables to the new scheme, and updates formulas using summary tables with a simple regex.

Test Plan:
Updated many tests to use the new style of name.

Added new Python tests to for resolving conflicts when renaming source tables and groupby columns.

Added a test for the migration, including renames in formulas.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3508
This commit is contained in:
Alex Hall
2022-07-11 20:00:25 +02:00
parent f1df6c0a46
commit b8486dcdba
18 changed files with 507 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

326
sandbox/grist/test_summary2.py
View File

@@ -4,9 +4,9 @@ files: test_summary.py and test_summary2.py.
"""
import actions
import logger
import objtypes
import test_engine
import test_summary
import testutil
from test_engine import Table, Column, View, Section, Field
@@ -32,15 +32,15 @@ class TestSummary2(test_engine.EngineTestCase):
# Check that we cannot add a non-formula column.
with self.assertRaisesRegex(ValueError, r'non-formula column'):
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "average",
self.apply_user_action(["AddColumn", "Address_summary_city_state", "average",
{"type": "Text", "isFormula": False}])
# Add two formula columns: one for 'state' (an existing column name, and a group-by column in
# some tables), and one for 'average' (a new column name).
self.apply_user_action(["AddVisibleColumn", "GristSummary_7_Address2", "state",
self.apply_user_action(["AddVisibleColumn", "Address_summary", "state",
{"formula": "':'.join(sorted(set($group.state)))"}])
self.apply_user_action(["AddVisibleColumn", "GristSummary_7_Address", "average",
self.apply_user_action(["AddVisibleColumn", "Address_summary_city_state", "average",
{"formula": "$amount / $count"}])
# Add two more summary tables: by 'city', and by 'state', and see what columns they get.
@@ -52,7 +52,7 @@ class TestSummary2(test_engine.EngineTestCase):
# Check the table and columns for all the summary tables.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_state", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "state", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -60,13 +60,13 @@ class TestSummary2(test_engine.EngineTestCase):
Column(18, "amount", "Numeric", True, "SUM($group.amount)", 0),
Column(23, "average", "Any", True, "$amount / $count", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary", 0, 1, columns=[
Column(19, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "count", "Int", True, "len($group)", 0),
Column(21, "amount", "Numeric", True, "SUM($group.amount)", 0),
Column(22, "state", "Any", True, "':'.join(sorted(set($group.state)))", 0),
]),
Table(4, "GristSummary_7_Address3", 0, 1, columns=[
Table(4, "Address_summary_city", 0, 1, columns=[
Column(24, "city", "Text", False, "", 11),
Column(25, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(26, "count", "Int", True, "len($group)", 0),
@@ -74,7 +74,7 @@ class TestSummary2(test_engine.EngineTestCase):
Column(28, "amount", "Numeric", True, "SUM($group.amount)", 0),
]),
# Note that since 'state' is used as a group-by column here, we skip the 'state' formula.
Table(5, "GristSummary_7_Address4", 0, 1, columns=[
Table(5, "Address_summary_state", 0, 1, columns=[
Column(29, "state", "Text", False, "", 12),
Column(30, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(31, "count", "Int", True, "len($group)", 0),
@@ -106,7 +106,7 @@ class TestSummary2(test_engine.EngineTestCase):
# Check that the data is as we expect.
self.assertTableData('GristSummary_7_Address', cols="all", data=[
self.assertTableData('Address_summary_city_state', cols="all", data=[
[ "id", "city", "state", "group", "count", "amount", "average" ],
[ 1, "New York", "NY" , [21,26,31],3, 1.+6+11 , (1.+6+11)/3 ],
[ 2, "Albany", "NY" , [22], 1, 2. , 2. ],
@@ -118,11 +118,11 @@ class TestSummary2(test_engine.EngineTestCase):
[ 8, "Boston", "MA" , [29], 1, 9. , 9. ],
[ 9, "Yonkers", "NY" , [30], 1, 10. , 10. ],
])
self.assertTableData('GristSummary_7_Address2', cols="all", data=[
self.assertTableData('Address_summary', cols="all", data=[
[ "id", "count", "amount", "state" , "group" ],
[ 1, 11, 66.0 , "IL:MA:NY:WA" , [21,22,23,24,25,26,27,28,29,30,31]],
])
self.assertTableData('GristSummary_7_Address3', cols="subset", data=[
self.assertTableData('Address_summary_city', cols="subset", data=[
[ "id", "city", "count", "amount", "state" ],
[ 1, "New York", 3, 1.+6+11 , "NY" ],
[ 2, "Albany", 1, 2. , "NY" ],
@@ -133,7 +133,7 @@ class TestSummary2(test_engine.EngineTestCase):
[ 7, "Boston", 1, 9. , "MA" ],
[ 8, "Yonkers", 1, 10. , "NY" ],
])
self.assertTableData('GristSummary_7_Address4', cols="subset", data=[
self.assertTableData('Address_summary_state', cols="subset", data=[
[ "id", "state", "count", "amount" ],
[ 1, "NY", 7, 1.+2+6+7+8+10+11 ],
[ 2, "WA", 1, 3. ],
@@ -146,16 +146,16 @@ class TestSummary2(test_engine.EngineTestCase):
self.assertPartialOutActions(out_actions, {
"stored": [
actions.UpdateRecord("Address", 28, {'state': 'MA'}),
actions.RemoveRecord("GristSummary_7_Address", 7),
actions.UpdateRecord("GristSummary_7_Address", 5, {'amount': 5.0 + 8.0}),
actions.UpdateRecord("GristSummary_7_Address", 5, {'average': 6.5}),
actions.UpdateRecord("GristSummary_7_Address", 5, {'count': 2}),
actions.UpdateRecord("GristSummary_7_Address", 5, {'group': [25, 28]}),
actions.UpdateRecord("GristSummary_7_Address3", 5, {'state': "MA"}),
actions.BulkUpdateRecord("GristSummary_7_Address4", [1,4],
actions.RemoveRecord("Address_summary_city_state", 7),
actions.UpdateRecord("Address_summary_city", 5, {'state': "MA"}),
actions.UpdateRecord("Address_summary_city_state", 5, {'amount': 5.0 + 8.0}),
actions.UpdateRecord("Address_summary_city_state", 5, {'average': 6.5}),
actions.UpdateRecord("Address_summary_city_state", 5, {'count': 2}),
actions.UpdateRecord("Address_summary_city_state", 5, {'group': [25, 28]}),
actions.BulkUpdateRecord("Address_summary_state", [1,4],
{'amount': [1.+2+6+7+10+11, 5.+8+9]}),
actions.BulkUpdateRecord("GristSummary_7_Address4", [1,4], {'count': [6, 3]}),
actions.BulkUpdateRecord("GristSummary_7_Address4", [1,4],
actions.BulkUpdateRecord("Address_summary_state", [1,4], {'count': [6, 3]}),
actions.BulkUpdateRecord("Address_summary_state", [1,4],
{'group': [[21,22,26,27,30,31], [25,28,29]]}),
]
})
@@ -175,11 +175,11 @@ class TestSummary2(test_engine.EngineTestCase):
# Check that we cannot rename a summary group-by column. (Perhaps it's better to raise an
# exception, but currently we translate the invalid request to a no-op.)
with self.assertRaisesRegex(ValueError, r'Cannot modify .* group-by'):
self.apply_user_action(["RenameColumn", "GristSummary_7_Address", "state", "s"])
self.apply_user_action(["RenameColumn", "Address_summary_city_state", "state", "s"])
# Verify all data. We'll repeat this after renamings to make sure there are no errors.
self.assertTableData("Address", self.starting_table_data)
self.assertTableData('GristSummary_7_Address', cols="all", data=[
self.assertTableData('Address_summary_city_state', cols="all", data=[
[ "id", "city", "state", "group", "count", "amount" ],
[ 1, "New York", "NY" , [21,26,31],3, 1.+6+11 ],
[ 2, "Albany", "NY" , [22], 1, 2. ],
@@ -191,25 +191,25 @@ class TestSummary2(test_engine.EngineTestCase):
[ 8, "Boston", "MA" , [29], 1, 9. ],
[ 9, "Yonkers", "NY" , [30], 1, 10. ],
])
self.assertTableData('GristSummary_7_Address2', cols="all", data=[
self.assertTableData('Address_summary', cols="all", data=[
[ "id", "count", "amount", "group" ],
[ 1, 11, 66.0 , [21,22,23,24,25,26,27,28,29,30,31]],
])
# This should work fine, and should affect sister tables.
self.apply_user_action(["RenameColumn", "GristSummary_7_Address", "count", "xcount"])
self.apply_user_action(["RenameColumn", "Address_summary_city_state", "count", "xcount"])
# These are the tables and columns we automatically get.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_state", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "state", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(17, "xcount", "Int", True, "len($group)", 0),
Column(18, "amount", "Numeric", True, "SUM($group.amount)", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary", 0, 1, columns=[
Column(19, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "xcount", "Int", True, "len($group)", 0),
Column(21, "amount", "Numeric", True, "SUM($group.amount)", 0),
@@ -226,14 +226,14 @@ class TestSummary2(test_engine.EngineTestCase):
Column(12, "xstate", "Text", False, "", 0),
Column(13, "xamount", "Numeric", False, "", 0),
]),
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_xstate", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "xstate", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(17, "xcount", "Int", True, "len($group)", 0),
Column(18, "xamount", "Numeric", True, "SUM($group.xamount)", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary", 0, 1, columns=[
Column(19, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "xcount", "Int", True, "len($group)", 0),
Column(21, "xamount", "Numeric", True, "SUM($group.xamount)", 0),
@@ -247,7 +247,7 @@ class TestSummary2(test_engine.EngineTestCase):
address_table_data = replace_col_names(
self.starting_table_data, state='xstate', amount='xamount')
self.assertTableData("Address", address_table_data)
self.assertTableData('GristSummary_7_Address', cols="all", data=[
self.assertTableData('Address_summary_city_xstate', cols="all", data=[
[ "id", "city", "xstate", "group", "xcount", "xamount" ],
[ 1, "New York", "NY" , [21,26,31],3, 1.+6+11 ],
[ 2, "Albany", "NY" , [22], 1, 2. ],
@@ -259,14 +259,14 @@ class TestSummary2(test_engine.EngineTestCase):
[ 8, "Boston", "MA" , [29], 1, 9. ],
[ 9, "Yonkers", "NY" , [30], 1, 10. ],
])
self.assertTableData('GristSummary_7_Address2', cols="all", data=[
self.assertTableData('Address_summary', cols="all", data=[
[ "id", "xcount", "xamount", "group" ],
[ 1, 11, 66.0 , [21,22,23,24,25,26,27,28,29,30,31]],
])
# Add a conflicting name to a summary table and see how renames behave.
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo",
self.apply_user_action(["AddColumn", "Address_summary_city_xstate", "foo",
{"formula": "$xamount * 100"}])
self.apply_user_action(["RenameColumn", "Address", "xstate", "foo"])
self.apply_user_action(["RenameColumn", "Address", "xamount", "foo"])
@@ -278,7 +278,7 @@ class TestSummary2(test_engine.EngineTestCase):
Column(12, "foo2", "Text", False, "", 0),
Column(13, "foo3", "Numeric", False, "", 0),
]),
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_foo2", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "foo2", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -286,7 +286,7 @@ class TestSummary2(test_engine.EngineTestCase):
Column(18, "foo3", "Numeric", True, "SUM($group.foo3)", 0),
Column(22, "foo", "Any", True, "$foo3 * 100", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary", 0, 1, columns=[
Column(19, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "xcount", "Int", True, "len($group)", 0),
Column(21, "foo3", "Numeric", True, "SUM($group.foo3)", 0),
@@ -297,7 +297,7 @@ class TestSummary2(test_engine.EngineTestCase):
address_table_data = replace_col_names(
address_table_data, xstate='foo2', xamount='foo3')
self.assertTableData("Address", address_table_data)
self.assertTableData('GristSummary_7_Address', cols="all", data=[
self.assertTableData('Address_summary_city_foo2', cols="all", data=[
[ "id", "city", "foo2" , "group", "xcount", "foo3", "foo" ],
[ 1, "New York", "NY" , [21,26,31],3, 1.+6+11, 100*(1.+6+11) ],
[ 2, "Albany", "NY" , [22], 1, 2. , 100*(2.) ],
@@ -309,7 +309,7 @@ class TestSummary2(test_engine.EngineTestCase):
[ 8, "Boston", "MA" , [29], 1, 9. , 100*(9.) ],
[ 9, "Yonkers", "NY" , [30], 1, 10. , 100*(10.) ],
])
self.assertTableData('GristSummary_7_Address2', cols="all", data=[
self.assertTableData('Address_summary', cols="all", data=[
[ "id", "xcount", "foo3" , "group" ],
[ 1, 11, 66.0 , [21,22,23,24,25,26,27,28,29,30,31]],
])
@@ -328,6 +328,145 @@ class TestSummary2(test_engine.EngineTestCase):
[21, 'foo3', True, 'WidgetOptions2'],
], rows=lambda r: r.colId in ('foo2', 'foo3'))
def test_summary_col_rename_conflict(self):
sample = testutil.parse_test_sample({
"SCHEMA": [
[1, "Table1", [
[11, "A", "Text", False, "", "A", ""],
[12, "B", "Text", False, "", "B", ""],
]],
[2, "Table1_summary_A_B", [
[13, "A", "Text", False, "", "A", ""],
]],
],
"DATA": {}
})
self.load_sample(sample)
self.apply_user_action(["CreateViewSection", 1, 0, "record", [11, 12], None])
self.apply_user_action(["CreateViewSection", 1, 0, "record", [11], None])
table1 = Table(
1, "Table1", primaryViewId=0, summarySourceTable=0, columns=[
Column(11, "A", "Text", False, "", 0),
Column(12, "B", "Text", False, "", 0),
],
)
# Normal table whose name conflicts with the automatically-generated summary table name below
fake_summary = Table(
2, "Table1_summary_A_B", primaryViewId=0, summarySourceTable=0, columns=[
Column(13, "A", "Text", False, "", 0),
],
)
# Auto-generated name has to have a '2' to disambiguate from the normal table.
summary_by_a_and_b = Table(
3, "Table1_summary_A_B2", primaryViewId=0, summarySourceTable=1, columns=[
Column(14, "A", "Text", False, "", 11),
Column(15, "B", "Text", False, "", 12),
Column(16, "group", "RefList:Table1", True, "table.getSummarySourceGroup(rec)", 0),
Column(17, "count", "Int", True, "len($group)", 0),
],
)
# nothing special here yet
summary_by_a = Table(
4, "Table1_summary_A", primaryViewId=0, summarySourceTable=1, columns=[
Column(18, "A", "Text", False, "", 11),
Column(19, "group", "RefList:Table1", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "count", "Int", True, "len($group)", 0),
],
)
tables = [table1, fake_summary, summary_by_a_and_b, summary_by_a]
self.assertTables(tables)
# Add some formulas using summary table names that are about to change
self.add_column("Table1", "summary_ref1",
type="RefList:Table1_summary_A_B2",
formula="Table1_summary_A_B2.lookupRecords(A=1)",
isFormula=True)
self.add_column("Table1", "summary_ref2",
type="Ref:Table1_summary_A",
formula="Table1_summary_A.lookupOne(A=23)",
isFormula=True)
# I got the weirdest heisenbug ever when renaming straight from A to A_B.
# The order of renaming is not deterministic so it may end up with
# 'Table1_summary_A_B3', but asserting that name made it come out as
# 'Table1_summary_A_B2' instead. Seems that file contents play a role in
# order in sets/dictionaries?
self.apply_user_action(["RenameColumn", "Table1", "A", "A2"])
self.apply_user_action(["RenameColumn", "Table1", "A2", "A_B"])
# Summary tables are automatically renamed to match the new column names.
summary_by_a_and_b = summary_by_a_and_b._replace(tableId="Table1_summary_A_B_B")
summary_by_a = summary_by_a._replace(tableId="Table1_summary_A_B2")
table1.columns[0] = table1.columns[0]._replace(colId="A_B")
summary_by_a_and_b.columns[0] = summary_by_a_and_b.columns[0]._replace(colId="A_B")
summary_by_a.columns[0] = summary_by_a.columns[0]._replace(colId="A_B")
table1.columns.extend([
Column(21, "summary_ref1", "RefList:Table1_summary_A_B_B", True,
"Table1_summary_A_B_B.lookupRecords(A_B=1)", 0),
Column(22, "summary_ref2", "Ref:Table1_summary_A_B2", True,
"Table1_summary_A_B2.lookupOne(A_B=23)", 0),
])
tables = [table1, fake_summary, summary_by_a_and_b, summary_by_a]
self.assertTables(tables)
def test_source_table_rename_conflict(self):
sample = testutil.parse_test_sample({
"SCHEMA": [
[1, "Table1", [
[11, "A", "Text", False, "", "A", ""],
]],
[2, "Table2_summary", [
[13, "A", "Text", False, "", "A", ""],
]],
],
"DATA": {}
})
self.load_sample(sample)
self.apply_user_action(["CreateViewSection", 1, 0, "record", [], None])
table1 = Table(
1, "Table1", primaryViewId=0, summarySourceTable=0, columns=[
Column(11, "A", "Text", False, "", 0),
],
)
fake_summary = Table(
2, "Table2_summary", primaryViewId=0, summarySourceTable=0, columns=[
Column(13, "A", "Text", False, "", 0),
],
)
summary = Table(
3, "Table1_summary", primaryViewId=0, summarySourceTable=1, columns=[
Column(14, "group", "RefList:Table1", True, "table.getSummarySourceGroup(rec)", 0),
Column(15, "count", "Int", True, "len($group)", 0),
],
)
tables = [table1, fake_summary, summary]
self.assertTables(tables)
self.apply_user_action(["RenameTable", "Table1", "Table2"])
table1 = table1._replace(tableId="Table2")
# Summary table is automatically renamed to match the new table name.
# Needs a '2' to disambiguate from the fake_summary table.
summary = summary._replace(tableId="Table2_summary2")
summary.columns[0] = summary.columns[0]._replace(type="RefList:Table2")
tables = [table1, fake_summary, summary]
self.assertTables(tables)
#----------------------------------------------------------------------
def test_restrictions(self):
@@ -342,7 +481,7 @@ class TestSummary2(test_engine.EngineTestCase):
self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])
self.apply_user_action(["CreateViewSection", 1, 0, "record", [], None])
self.assertTableData('GristSummary_7_Address', cols="all", data=[
self.assertTableData('Address_summary_city_state', cols="all", data=[
[ "id", "city", "state", "group", "count", "amount" ],
[ 1, "New York", "NY" , [21,26,31],3, 1.+6+11 ],
[ 2, "Albany", "NY" , [22], 1, 2. ],
@@ -357,48 +496,49 @@ class TestSummary2(test_engine.EngineTestCase):
# (1) no adding/removing/renaming non-formula columns.
with self.assertRaisesRegex(ValueError, r'non-formula column'):
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo",
self.apply_user_action(["AddColumn", "Address_summary_city_state", "foo",
{"type": "Numeric", "isFormula": False}])
with self.assertRaisesRegex(ValueError, r'group-by column'):
self.apply_user_action(["RemoveColumn", "GristSummary_7_Address", "state"])
self.apply_user_action(["RemoveColumn", "Address_summary_city_state", "state"])
with self.assertRaisesRegex(ValueError, r'Cannot modify .* group-by'):
self.apply_user_action(["RenameColumn", "GristSummary_7_Address", "state", "st"])
self.apply_user_action(["RenameColumn", "Address_summary_city_state", "state", "st"])
# (2) no converting between formula/non-formula
with self.assertRaisesRegex(ValueError, r'Cannot change .* formula and data'):
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "amount",
self.apply_user_action(["ModifyColumn", "Address_summary_city_state", "amount",
{"isFormula": False}])
with self.assertRaisesRegex(ValueError, r'Cannot change .* formula and data'):
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "state",
self.apply_user_action(["ModifyColumn", "Address_summary_city_state", "state",
{"isFormula": True}])
# (3) no editing values in non-formula columns
with self.assertRaisesRegex(ValueError, r'Cannot enter data .* group-by'):
self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 6, {"state": "ny"}])
self.apply_user_action(["UpdateRecord", "Address_summary_city_state", 6, {"state": "ny"}])
# (4) no removing rows (this is questionable b/c empty rows might be OK to remove)
with self.assertRaisesRegex(ValueError, r'Cannot remove record .* summary'):
self.apply_user_action(["RemoveRecord", "GristSummary_7_Address", 6])
self.apply_user_action(["RemoveRecord", "Address_summary_city_state", 6])
# (5) no renaming summary tables.
with self.assertRaisesRegex(ValueError, r'cannot rename .* summary'):
self.apply_user_action(["RenameTable", "GristSummary_7_Address", "GristSummary_hello"])
self.apply_user_action(["RenameTable", "Address_summary_city_state", "Address_summary_X"])
# Check that we can add an empty column, then set a formula for it.
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo", {}])
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "foo", {"formula": "1+1"}])
self.apply_user_action(["AddColumn", "Address_summary_city_state", "foo", {}])
self.apply_user_action(["ModifyColumn", "Address_summary_city_state", "foo",
{"formula": "1+1"}])
with self.assertRaisesRegex(ValueError, "Can't save .* to formula"):
self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 1, {"foo": "hello"}])
self.apply_user_action(["UpdateRecord", "Address_summary_city_state", 1, {"foo": "hello"}])
# But we cannot add an empty column, then add a value to it.
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo2", {}])
self.apply_user_action(["AddColumn", "Address_summary_city_state", "foo2", {}])
with self.assertRaisesRegex(ValueError, r'Cannot change .* between formula and data'):
self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 1, {"foo2": "hello"}])
self.apply_user_action(["UpdateRecord", "Address_summary_city_state", 1, {"foo2": "hello"}])
self.assertTableData('GristSummary_7_Address', cols="all", data=[
self.assertTableData('Address_summary_city_state', cols="all", data=[
[ "id", "city", "state", "group", "count", "amount", "foo", "foo2" ],
[ 1, "New York", "NY" , [21,26,31],3, 1.+6+11 , 2 , None ],
[ 2, "Albany", "NY" , [22], 1, 2. , 2 , None ],
@@ -426,7 +566,7 @@ class TestSummary2(test_engine.EngineTestCase):
# We should have a single summary table, and a single section referring to it.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_state", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "state", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -442,7 +582,7 @@ class TestSummary2(test_engine.EngineTestCase):
Field(8, colRef=18),
])
])])
self.assertEqual(get_helper_cols('Address'), ['#summary#GristSummary_7_Address'])
self.assertEqual(get_helper_cols('Address'), ['#summary#Address_summary_city_state'])
# Verify more fields of some of the new column objects.
self.assertTableData('_grist_Tables_column', rows="subset", cols="subset", data=[
@@ -457,7 +597,7 @@ class TestSummary2(test_engine.EngineTestCase):
self.assertTables([
self.starting_table,
# Note that Table #2 is gone at this point, since it's unused.
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary_state", 0, 1, columns=[
Column(19, "state", "Text", False, "", 12),
Column(20, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(21, "count", "Int", True, "len($group)", 0),
@@ -471,14 +611,14 @@ class TestSummary2(test_engine.EngineTestCase):
Field(8, colRef=22),
])
])])
self.assertTableData('GristSummary_7_Address2', cols="subset", data=[
self.assertTableData('Address_summary_state', cols="subset", data=[
[ "id", "state", "count", "amount" ],
[ 1, "NY", 7, 1.+2+6+7+8+10+11 ],
[ 2, "WA", 1, 3. ],
[ 3, "IL", 1, 4. ],
[ 4, "MA", 2, 5.+9 ],
])
self.assertEqual(get_helper_cols('Address'), ['#summary#GristSummary_7_Address2'])
self.assertEqual(get_helper_cols('Address'), ['#summary#Address_summary_state'])
# Verify more fields of some of the new column objects.
self.assertTableData('_grist_Tables_column', rows="subset", cols="subset", data=[
@@ -492,7 +632,7 @@ class TestSummary2(test_engine.EngineTestCase):
self.assertTables([
self.starting_table,
# Note that Table #3 is gone at this point, since it's unused.
Table(4, "GristSummary_7_Address", 0, 1, columns=[
Table(4, "Address_summary_city", 0, 1, columns=[
Column(23, "city", "Text", False, "", 11),
Column(24, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(25, "count", "Int", True, "len($group)", 0),
@@ -506,7 +646,7 @@ class TestSummary2(test_engine.EngineTestCase):
Field(8, colRef=26),
])
])])
self.assertTableData('GristSummary_7_Address', cols="subset", data=[
self.assertTableData('Address_summary_city', cols="subset", data=[
[ "id", "city", "count", "amount" ],
[ 1, "New York", 3, 1.+6+11 ],
[ 2, "Albany", 1, 2. ],
@@ -517,7 +657,7 @@ class TestSummary2(test_engine.EngineTestCase):
[ 7, "Boston", 1, 9. ],
[ 8, "Yonkers", 1, 10. ],
])
self.assertEqual(get_helper_cols('Address'), ['#summary#GristSummary_7_Address'])
self.assertEqual(get_helper_cols('Address'), ['#summary#Address_summary_city'])
# Verify more fields of some of the new column objects.
self.assertTableData('_grist_Tables_column', rows="subset", cols="subset", data=[
@@ -531,7 +671,7 @@ class TestSummary2(test_engine.EngineTestCase):
self.assertTables([
self.starting_table,
# Note that Table #4 is gone at this point, since it's unused.
Table(5, "GristSummary_7_Address2", 0, 1, columns=[
Table(5, "Address_summary", 0, 1, columns=[
Column(27, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(28, "count", "Int", True, "len($group)", 0),
Column(29, "amount", "Numeric", True, "SUM($group.amount)", 0),
@@ -543,18 +683,18 @@ class TestSummary2(test_engine.EngineTestCase):
Field(8, colRef=29),
])
])])
self.assertTableData('GristSummary_7_Address2', cols="subset", data=[
self.assertTableData('Address_summary', cols="subset", data=[
[ "id", "count", "amount"],
[ 1, 11, 66.0 ],
])
self.assertEqual(get_helper_cols('Address'), ['#summary#GristSummary_7_Address2'])
self.assertEqual(get_helper_cols('Address'), ['#summary#Address_summary'])
# Back to full circle, but with group-by columns differently arranged.
self.apply_user_action(["UpdateSummaryViewSection", 2, [12,11]])
self.assertTables([
self.starting_table,
# Note that Table #5 is gone at this point, since it's unused.
Table(6, "GristSummary_7_Address", 0, 1, columns=[
Table(6, "Address_summary_city_state", 0, 1, columns=[
Column(30, "state", "Text", False, "", 12),
Column(31, "city", "Text", False, "", 11),
Column(32, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -570,7 +710,7 @@ class TestSummary2(test_engine.EngineTestCase):
Field(8, colRef=34),
])
])])
self.assertTableData('GristSummary_7_Address', cols="subset", data=[
self.assertTableData('Address_summary_city_state', cols="subset", data=[
[ "id", "city", "state", "count", "amount" ],
[ 1, "New York", "NY" , 3, 1.+6+11 ],
[ 2, "Albany", "NY" , 1, 2. ],
@@ -582,13 +722,13 @@ class TestSummary2(test_engine.EngineTestCase):
[ 8, "Boston", "MA" , 1, 9. ],
[ 9, "Yonkers", "NY" , 1, 10. ],
])
self.assertEqual(get_helper_cols('Address'), ['#summary#GristSummary_7_Address'])
self.assertEqual(get_helper_cols('Address'), ['#summary#Address_summary_city_state'])
# Now add a different view section with the same group-by columns.
self.apply_user_action(["CreateViewSection", 1, 1, "record", [11,12], None])
self.assertTables([
self.starting_table,
Table(6, "GristSummary_7_Address", 0, 1, columns=[
Table(6, "Address_summary_city_state", 0, 1, columns=[
Column(30, "state", "Text", False, "", 12),
Column(31, "city", "Text", False, "", 11),
Column(32, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -610,20 +750,20 @@ class TestSummary2(test_engine.EngineTestCase):
Field(27, colRef=34),
])
])])
self.assertEqual(get_helper_cols('Address'), ['#summary#GristSummary_7_Address'])
self.assertEqual(get_helper_cols('Address'), ['#summary#Address_summary_city_state'])
# Change one view section, and ensure there are now two summary tables.
self.apply_user_action(["UpdateSummaryViewSection", 7, []])
self.assertTables([
self.starting_table,
Table(6, "GristSummary_7_Address", 0, 1, columns=[
Table(6, "Address_summary_city_state", 0, 1, columns=[
Column(30, "state", "Text", False, "", 12),
Column(31, "city", "Text", False, "", 11),
Column(32, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(33, "count", "Int", True, "len($group)", 0),
Column(34, "amount", "Numeric", True, "SUM($group.amount)", 0),
]),
Table(7, "GristSummary_7_Address2", 0, 1, columns=[
Table(7, "Address_summary", 0, 1, columns=[
Column(35, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(36, "count", "Int", True, "len($group)", 0),
Column(37, "amount", "Numeric", True, "SUM($group.amount)", 0),
@@ -641,15 +781,15 @@ class TestSummary2(test_engine.EngineTestCase):
Field(27, colRef=37),
])
])])
self.assertEqual(get_helper_cols('Address'), ['#summary#GristSummary_7_Address',
'#summary#GristSummary_7_Address2'])
self.assertEqual(get_helper_cols('Address'), ['#summary#Address_summary_city_state',
'#summary#Address_summary'])
# Delete one view section, and see that the summary table is gone.
self.apply_user_action(["RemoveViewSection", 7])
self.assertTables([
self.starting_table,
# Note that Table #7 is gone at this point, since it's now unused.
Table(6, "GristSummary_7_Address", 0, 1, columns=[
Table(6, "Address_summary_city_state", 0, 1, columns=[
Column(30, "state", "Text", False, "", 12),
Column(31, "city", "Text", False, "", 11),
Column(32, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -665,14 +805,14 @@ class TestSummary2(test_engine.EngineTestCase):
Field(8, colRef=34),
])
])])
self.assertEqual(get_helper_cols('Address'), ['#summary#GristSummary_7_Address'])
self.assertEqual(get_helper_cols('Address'), ['#summary#Address_summary_city_state'])
# Change the section to add and then remove the "amount" to the group-by column; check that
# column "amount" was correctly restored
self.apply_user_action(["UpdateSummaryViewSection", 2, [11, 12, 13]])
self.assertTables([
self.starting_table,
Table(7, "GristSummary_7_Address2", 0, 1, columns=[
Table(7, "Address_summary_amount_city_state", 0, 1, columns=[
Column(35, "city", "Text", False, "", 11),
Column(36, "state", "Text", False, "", 12),
Column(37, "amount", "Numeric", False, "", 13),
@@ -691,7 +831,7 @@ class TestSummary2(test_engine.EngineTestCase):
self.apply_user_action(["UpdateSummaryViewSection", 2, [11,12]])
self.assertTables([
self.starting_table,
Table(8, "GristSummary_7_Address", 0, 1, columns=[
Table(8, "Address_summary_city_state", 0, 1, columns=[
Column(40, "city", "Text", False, "", 11),
Column(41, "state", "Text", False, "", 12),
Column(42, "amount", "Numeric", True, "SUM($group.amount)", 0),
@@ -715,7 +855,7 @@ class TestSummary2(test_engine.EngineTestCase):
self.apply_user_action(["UpdateSummaryViewSection", 2, [11]])
self.assertTables([
self.starting_table,
Table(9, "GristSummary_7_Address2", 0, 1, columns=[
Table(9, "Address_summary_city", 0, 1, columns=[
Column(45, "city", "Text", False, "", 11),
Column(46, "amount", "Numeric", True, "SUM($group.amount)", 0),
Column(48, "count", "Int", True, "len($group)", 0),
@@ -741,13 +881,13 @@ class TestSummary2(test_engine.EngineTestCase):
self.load_sample(self.sample)
self.apply_user_action(["CreateViewSection", 1, 0, "record", [12], None])
self.apply_user_action(["AddVisibleColumn", "GristSummary_7_Address", "city",
self.apply_user_action(["AddVisibleColumn", "Address_summary_state", "city",
{"formula": "$state.lower()"}])
# We should have a single summary table, and a single section referring to it.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_state", 0, 1, columns=[
Column(14, "state", "Text", False, "", 12),
Column(15, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(16, "count", "Int", True, "len($group)", 0),
@@ -763,7 +903,7 @@ class TestSummary2(test_engine.EngineTestCase):
Field(8, colRef=18),
])
])])
self.assertTableData('GristSummary_7_Address', cols="subset", data=[
self.assertTableData('Address_summary_state', cols="subset", data=[
[ "id", "state", "count", "amount" , "city"],
[ 1, "NY", 7, 1.+2+6+7+8+10+11 , "ny" ],
[ 2, "WA", 1, 3. , "wa" ],
@@ -775,7 +915,7 @@ class TestSummary2(test_engine.EngineTestCase):
self.apply_user_action(["UpdateSummaryViewSection", 2, [11,12]])
self.assertTables([
self.starting_table,
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary_city_state", 0, 1, columns=[
Column(19, "city", "Text", False, "", 11),
Column(20, "state", "Text", False, "", 12),
Column(21, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -812,19 +952,19 @@ class TestSummary2(test_engine.EngineTestCase):
# We should have a single summary table, and a single section referring to it.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_state", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "state", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(17, "count", "Int", True, "len($group)", 0),
Column(18, "amount", "Numeric", True, "SUM($group.amount)", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary", 0, 1, columns=[
Column(19, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "count", "Int", True, "len($group)", 0),
Column(21, "amount", "Numeric", True, "SUM($group.amount)", 0),
]),
Table(4, "GristSummary_7_Address3", 0, 1, columns=[
Table(4, "Address_summary_state", 0, 1, columns=[
Column(22, "state", "Text", False, "", 12),
Column(23, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(24, "count", "Int", True, "len($group)", 0),
@@ -862,7 +1002,7 @@ class TestSummary2(test_engine.EngineTestCase):
# Verify that unused summary tables are also gone, but the one used remains.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_state", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "state", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -893,7 +1033,7 @@ class TestSummary2(test_engine.EngineTestCase):
# We should have a single summary table, and a single (non-raw) section referring to it.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_state", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "state", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -912,7 +1052,7 @@ class TestSummary2(test_engine.EngineTestCase):
self.assertTables([
self.starting_table,
# Note that Table #2 is gone at this point, since it's unused.
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary_state", 0, 1, columns=[
Column(19, "state", "Text", False, "", 12),
Column(20, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(21, "count", "Int", True, "len($group)", 0),
@@ -935,13 +1075,13 @@ class TestSummary2(test_engine.EngineTestCase):
self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])
self.apply_user_action(["CreateViewSection", 1, 0, "record", [], None])
# Add a formula column
self.apply_user_action(["AddVisibleColumn", "GristSummary_7_Address", "average",
self.apply_user_action(["AddVisibleColumn", "Address_summary_city_state", "average",
{"formula": "$amount / $count"}])
# Check the table and columns for all the summary tables.
self.assertTables([
self.starting_table,
Table(2, "GristSummary_7_Address", 0, 1, columns=[
Table(2, "Address_summary_city_state", 0, 1, columns=[
Column(14, "city", "Text", False, "", 11),
Column(15, "state", "Text", False, "", 12),
Column(16, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
@@ -949,7 +1089,7 @@ class TestSummary2(test_engine.EngineTestCase):
Column(18, "amount", "Numeric", True, "SUM($group.amount)", 0),
Column(22, "average", "Any", True, "$amount / $count", 0),
]),
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary", 0, 1, columns=[
Column(19, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "count", "Int", True, "len($group)", 0),
Column(21, "amount", "Numeric", True, "SUM($group.amount)", 0),
@@ -986,7 +1126,7 @@ class TestSummary2(test_engine.EngineTestCase):
# Check the table and columns for all the summary tables.
self.assertTables([
self.starting_table,
Table(3, "GristSummary_7_Address2", 0, 1, columns=[
Table(3, "Address_summary", 0, 1, columns=[
Column(19, "group", "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
Column(20, "count", "Int", True, "len($group)", 0),
Column(21, "amount", "Numeric", True, "SUM($group.amount)", 0),
@@ -1039,7 +1179,7 @@ class TestSummary2(test_engine.EngineTestCase):
[ 8, 8.0, "Boston", "MA" , [29], 1, 9. , 9. ],
[ 9, 9.0, "Yonkers", "NY" , [30], 1, 10. , 10. ],
])
self.assertTableData('GristSummary_7_Address2', cols="all", data=[
self.assertTableData('Address_summary', cols="all", data=[
[ "id", "count", "amount", "group" ],
[ 1, 11, 66.0 , [21,22,23,24,25,26,27,28,29,30,31]],
])
@@ -1069,7 +1209,7 @@ class TestSummary2(test_engine.EngineTestCase):
Column(24, "group", "RefList:Address", True,
"Address.lookupRecords(city=$city, state=$state)", 0),
]),
Table(4, "GristSummary_6_Table1", primaryViewId=0, summarySourceTable=3, columns=[
Table(4, "Table1_summary_state", primaryViewId=0, summarySourceTable=3, columns=[
Column(25, "state", "Text", False, "", 21),
Column(26, "group", "RefList:Table1", True, "table.getSummarySourceGroup(rec)", 0),
Column(27, "count", "Int", True, "SUM($group.count)", 0),
@@ -1090,7 +1230,7 @@ class TestSummary2(test_engine.EngineTestCase):
[ 8, 8.0, "Boston", "MA" , [29], 1, 9. ],
[ 9, 9.0, "Yonkers", "NY" , [30], 1, 10. ],
])
self.assertTableData('GristSummary_6_Table1', cols="all", data=[
self.assertTableData('Table1_summary_state', cols="all", data=[
[ "id", "state", "group", "count", "amount" ],
[ 1, "NY", [1,2,6,7,9], 7, 1.+6+11+2+7+8+10 ],
[ 2, "WA", [3], 1, 3. ],