mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
b8486dcdba
Summary: Changes auto-generated summary table IDs from e.g. `GristSummary_6_Table1` to `Table1_summary_A_B` (meaning `Table1` grouped by `A` and `B`). This makes it easier to write formulas involving summary tables, make API requests, understand logs, etc. Because these don't encode the source table ID as reliably as before, `decode_summary_table_name` now uses the summary table schema info, not just the summary table ID. Specifically, it looks at the type of the `group` column, which is `RefList:<source table id>`. Renaming a source table renames the summary table as before, and now renaming a groupby column renames the summary table as well. Conflicting table names are resolved in the usual way by adding a number at the end, e.g. `Table1_summary_A_B2`. These summary tables are not automatically renamed when the disambiguation is no longer needed. A new migration renames all summary tables to the new scheme, and updates formulas using summary tables with a simple regex. Test Plan: Updated many tests to use the new style of name. Added new Python tests to for resolving conflicts when renaming source tables and groupby columns. Added a test for the migration, including renames in formulas. Reviewers: georgegevoian Reviewed By: georgegevoian Differential Revision: https://phab.getgrist.com/D3508
455 lines
19 KiB
Python
455 lines
19 KiB
Python
import logger
|
|
|
|
import testutil
|
|
import test_engine
|
|
from test_engine import Table, Column, View, Section, Field
|
|
|
|
log = logger.Logger(__name__, logger.INFO)
|
|
|
|
class TestColumnActions(test_engine.EngineTestCase):
|
|
sample = testutil.parse_test_sample({
|
|
"SCHEMA": [
|
|
[1, "Address", [
|
|
[21, "city", "Text", False, "", "", ""],
|
|
]]
|
|
],
|
|
"DATA": {
|
|
"Address": [
|
|
["id", "city" ],
|
|
[11, "New York" ],
|
|
[12, "Colombia" ],
|
|
[13, "New Haven" ],
|
|
[14, "West Haven" ]],
|
|
}
|
|
})
|
|
|
|
@test_engine.test_undo
|
|
def test_column_updates(self):
|
|
# Verify various automatic adjustments for column updates
|
|
# (1) that label gets synced to colId unless untieColIdFromLabel is set.
|
|
# (2) that unsetting untieColId syncs the label to colId.
|
|
# (3) that a complex BulkUpdateRecord for _grist_Tables_column is processed correctly.
|
|
self.load_sample(self.sample)
|
|
|
|
self.apply_user_action(["AddColumn", "Address", "foo", {"type": "Numeric"}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "city", "", "Text", False ],
|
|
[ 22, 1, "foo", "foo", "Numeric", False ],
|
|
])
|
|
|
|
# Check that label is synced to colId, via either ModifyColumn or UpdateRecord useraction.
|
|
self.apply_user_action(["ModifyColumn", "Address", "city", {"label": "Hello"}])
|
|
self.apply_user_action(["UpdateRecord", "_grist_Tables_column", 22, {"label": "World"}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Hello", "Hello", "Text", False ],
|
|
[ 22, 1, "World", "World", "Numeric", False ],
|
|
])
|
|
|
|
# But check that a rename or an update that includes colId is not affected by label.
|
|
self.apply_user_action(["RenameColumn", "Address", "Hello", "Hola"])
|
|
self.apply_user_action(["UpdateRecord", "_grist_Tables_column", 22,
|
|
{"label": "Foo", "colId": "Bar"}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Hola", "Hello", "Text", False ],
|
|
[ 22, 1, "Bar", "Foo", "Numeric", False ],
|
|
])
|
|
|
|
# Check that setting untieColIdFromLabel doesn't change anything immediately.
|
|
self.apply_user_action(["BulkUpdateRecord", "_grist_Tables_column", [21,22],
|
|
{"untieColIdFromLabel": [True, True]}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Hola", "Hello", "Text", True ],
|
|
[ 22, 1, "Bar", "Foo", "Numeric", True ],
|
|
])
|
|
|
|
# Check that ModifyColumn and UpdateRecord useractions no longer copy label to colId.
|
|
self.apply_user_action(["ModifyColumn", "Address", "Hola", {"label": "Hello"}])
|
|
self.apply_user_action(["UpdateRecord", "_grist_Tables_column", 22, {"label": "World"}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Hola", "Hello", "Text", True ],
|
|
[ 22, 1, "Bar", "World", "Numeric", True ],
|
|
])
|
|
|
|
# Check that unsetting untieColIdFromLabel syncs label, whether label is provided or not.
|
|
self.apply_user_action(["UpdateRecord", "_grist_Tables_column", 21,
|
|
{"untieColIdFromLabel": False, "label": "Alice"}])
|
|
self.apply_user_action(["UpdateRecord", "_grist_Tables_column", 22,
|
|
{"untieColIdFromLabel": False}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Alice", "Alice", "Text", False ],
|
|
[ 22, 1, "World", "World", "Numeric", False ],
|
|
])
|
|
|
|
# Check that column names still get sanitized and disambiguated.
|
|
self.apply_user_action(["UpdateRecord", "_grist_Tables_column", 21, {"label": "Alice M"}])
|
|
self.apply_user_action(["UpdateRecord", "_grist_Tables_column", 22, {"label": "Alice-M"}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Alice_M", "Alice M", "Text", False ],
|
|
[ 22, 1, "Alice_M2", "Alice-M", "Numeric", False ],
|
|
])
|
|
|
|
# Check that a column rename doesn't avoid its own name.
|
|
self.apply_user_action(["UpdateRecord", "_grist_Tables_column", 21, {"label": "Alice*M"}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Alice_M", "Alice*M", "Text", False ],
|
|
[ 22, 1, "Alice_M2", "Alice-M", "Numeric", False ],
|
|
])
|
|
|
|
# Untie colIds and tie them again, and make sure it doesn't cause unneeded renames.
|
|
self.apply_user_action(["BulkUpdateRecord", "_grist_Tables_column", [21,22],
|
|
{ "untieColIdFromLabel": [True, True] }])
|
|
self.apply_user_action(["BulkUpdateRecord", "_grist_Tables_column", [21,22],
|
|
{ "untieColIdFromLabel": [False, False] }])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Alice_M", "Alice*M", "Text", False ],
|
|
[ 22, 1, "Alice_M2", "Alice-M", "Numeric", False ],
|
|
])
|
|
|
|
# Check that disambiguating also works correctly for bulk updates.
|
|
self.apply_user_action(["BulkUpdateRecord", "_grist_Tables_column", [21,22],
|
|
{"label": ["Bob Z", "Bob-Z"]}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Bob_Z", "Bob Z", "Text", False ],
|
|
[ 22, 1, "Bob_Z2", "Bob-Z", "Numeric", False ],
|
|
])
|
|
|
|
# Same for changing colIds directly.
|
|
self.apply_user_action(["BulkUpdateRecord", "_grist_Tables_column", [21,22],
|
|
{"colId": ["Carol X", "Carol-X"]}])
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Carol_X", "Bob Z", "Text", False ],
|
|
[ 22, 1, "Carol_X2", "Bob-Z", "Numeric", False ],
|
|
])
|
|
|
|
# Check confusing bulk updates with different keys changing for different records.
|
|
out_actions = self.apply_user_action(["BulkUpdateRecord", "_grist_Tables_column", [21,22], {
|
|
"label": ["Bob Z", "Bob-Z"], # Unchanged from before.
|
|
"untieColIdFromLabel": [True, False]
|
|
}])
|
|
self.assertPartialOutActions(out_actions, { "stored": [
|
|
["RenameColumn", "Address", "Carol_X2", "Bob_Z"],
|
|
["BulkUpdateRecord", "_grist_Tables_column", [21, 22],
|
|
{"colId": ["Carol_X", "Bob_Z"], # Note that only one column is changing.
|
|
"untieColIdFromLabel": [True, False]
|
|
# No update to label, they get trimmed as unchanged.
|
|
}
|
|
],
|
|
]})
|
|
self.assertTableData("_grist_Tables_column", cols="subset", data=[
|
|
[ "id", "parentId", "colId", "label", "type", "untieColIdFromLabel" ],
|
|
[ 21, 1, "Carol_X", "Bob Z", "Text", True ],
|
|
[ 22, 1, "Bob_Z", "Bob-Z", "Numeric", False ],
|
|
])
|
|
|
|
#----------------------------------------------------------------------
|
|
|
|
address_table_data = [
|
|
["id", "city", "state", "amount" ],
|
|
[ 21, "New York", "NY" , 1. ],
|
|
[ 22, "Albany", "NY" , 2. ],
|
|
[ 23, "Seattle", "WA" , 3. ],
|
|
[ 24, "Chicago", "IL" , 4. ],
|
|
[ 25, "Bedford", "MA" , 5. ],
|
|
[ 26, "New York", "NY" , 6. ],
|
|
[ 27, "Buffalo", "NY" , 7. ],
|
|
[ 28, "Bedford", "NY" , 8. ],
|
|
[ 29, "Boston", "MA" , 9. ],
|
|
[ 30, "Yonkers", "NY" , 10. ],
|
|
[ 31, "New York", "NY" , 11. ],
|
|
]
|
|
|
|
sample2 = testutil.parse_test_sample({
|
|
"SCHEMA": [
|
|
[1, "Address", [
|
|
[11, "city", "Text", False, "", "", ""],
|
|
[12, "state", "Text", False, "", "", ""],
|
|
[13, "amount", "Numeric", False, "", "", ""],
|
|
]]
|
|
],
|
|
"DATA": {
|
|
"Address": address_table_data
|
|
}
|
|
})
|
|
|
|
def init_sample_data(self):
|
|
# Add a new view with a section, and a new table to that view, and a summary table.
|
|
self.load_sample(self.sample2)
|
|
self.apply_user_action(["CreateViewSection", 1, 0, "record", None, None])
|
|
self.apply_user_action(["AddEmptyTable", None])
|
|
self.apply_user_action(["CreateViewSection", 2, 1, "record", None, None])
|
|
self.apply_user_action(["CreateViewSection", 1, 1, "record", [12], None])
|
|
self.apply_user_action(["BulkAddRecord", "Table1", [None]*3, {
|
|
"A": ["a", "b", "c"],
|
|
"B": ["d", "e", "f"],
|
|
"C": ["", "", ""]
|
|
}])
|
|
|
|
# Verify the new structure of tables and views.
|
|
self.assertTables([
|
|
Table(1, "Address", primaryViewId=0, summarySourceTable=0, columns=[
|
|
Column(11, "city", "Text", False, "", 0),
|
|
Column(12, "state", "Text", False, "", 0),
|
|
Column(13, "amount", "Numeric", False, "", 0),
|
|
]),
|
|
Table(2, "Table1", 2, 0, columns=[
|
|
Column(14, "manualSort", "ManualSortPos", False, "", 0),
|
|
Column(15, "A", "Text", False, "", 0),
|
|
Column(16, "B", "Text", False, "", 0),
|
|
Column(17, "C", "Any", True, "", 0),
|
|
]),
|
|
Table(3, "Address_summary_state", 0, 1, columns=[
|
|
Column(18, "state", "Text", False, "", summarySourceCol=12),
|
|
Column(19, "group", "RefList:Address", True, summarySourceCol=0,
|
|
formula="table.getSummarySourceGroup(rec)"),
|
|
Column(20, "count", "Int", True, summarySourceCol=0, formula="len($group)"),
|
|
Column(21, "amount", "Numeric", True, summarySourceCol=0, formula="SUM($group.amount)"),
|
|
]),
|
|
])
|
|
self.assertViews([
|
|
View(1, sections=[
|
|
Section(1, parentKey="record", tableRef=1, fields=[
|
|
Field(1, colRef=11),
|
|
Field(2, colRef=12),
|
|
Field(3, colRef=13),
|
|
]),
|
|
Section(4, parentKey="record", tableRef=2, fields=[
|
|
Field(10, colRef=15),
|
|
Field(11, colRef=16),
|
|
Field(12, colRef=17),
|
|
]),
|
|
Section(6, parentKey="record", tableRef=3, fields=[
|
|
Field(16, colRef=18),
|
|
Field(17, colRef=20),
|
|
Field(18, colRef=21),
|
|
]),
|
|
]),
|
|
View(2, sections=[
|
|
Section(2, parentKey="record", tableRef=2, fields=[
|
|
Field(4, colRef=15),
|
|
Field(5, colRef=16),
|
|
Field(6, colRef=17),
|
|
]),
|
|
])
|
|
])
|
|
self.assertTableData('Address', data=self.address_table_data)
|
|
self.assertTableData('Table1', data=[
|
|
["id", "A", "B", "C", "manualSort"],
|
|
[ 1, "a", "d", None, 1.0],
|
|
[ 2, "b", "e", None, 2.0],
|
|
[ 3, "c", "f", None, 3.0],
|
|
])
|
|
self.assertTableData("Address_summary_state", cols="subset", data=[
|
|
[ "id", "state", "count", "amount" ],
|
|
[ 1, "NY", 7, 1.+2+6+7+8+10+11 ],
|
|
[ 2, "WA", 1, 3. ],
|
|
[ 3, "IL", 1, 4. ],
|
|
[ 4, "MA", 2, 5.+9 ],
|
|
])
|
|
|
|
#----------------------------------------------------------------------
|
|
|
|
@test_engine.test_undo
|
|
def test_column_removals(self):
|
|
# Verify removal of fields when columns are removed.
|
|
|
|
self.init_sample_data()
|
|
|
|
# Add link{Src,Target}ColRef to ViewSections. These aren't actually meaningful links, but they
|
|
# should still get cleared automatically when columns get removed.
|
|
self.apply_user_action(['UpdateRecord', '_grist_Views_section', 2, {
|
|
'linkSrcSectionRef': 1,
|
|
'linkSrcColRef': 11,
|
|
'linkTargetColRef': 16
|
|
}])
|
|
self.assertTableData('_grist_Views_section', cols="subset", rows="subset", data=[
|
|
["id", "linkSrcSectionRef", "linkSrcColRef", "linkTargetColRef"],
|
|
[2, 1, 11, 16 ],
|
|
])
|
|
|
|
# Test that we can remove multiple columns using BulkUpdateRecord.
|
|
self.apply_user_action(["BulkRemoveRecord", '_grist_Tables_column', [11, 16]])
|
|
|
|
# Test that link{Src,Target}colRef back-references get unset.
|
|
self.assertTableData('_grist_Views_section', cols="subset", rows="subset", data=[
|
|
["id", "linkSrcSectionRef", "linkSrcColRef", "linkTargetColRef"],
|
|
[2, 1, 0, 0 ],
|
|
])
|
|
|
|
# Test that columns and section fields got removed.
|
|
self.assertTables([
|
|
Table(1, "Address", primaryViewId=0, summarySourceTable=0, columns=[
|
|
Column(12, "state", "Text", False, "", 0),
|
|
Column(13, "amount", "Numeric", False, "", 0),
|
|
]),
|
|
Table(2, "Table1", 2, 0, columns=[
|
|
Column(14, "manualSort", "ManualSortPos", False, "", 0),
|
|
Column(15, "A", "Text", False, "", 0),
|
|
Column(17, "C", "Any", True, "", 0),
|
|
]),
|
|
Table(3, "Address_summary_state", 0, 1, columns=[
|
|
Column(18, "state", "Text", False, "", summarySourceCol=12),
|
|
Column(19, "group", "RefList:Address", True, summarySourceCol=0,
|
|
formula="table.getSummarySourceGroup(rec)"),
|
|
Column(20, "count", "Int", True, summarySourceCol=0, formula="len($group)"),
|
|
Column(21, "amount", "Numeric", True, summarySourceCol=0, formula="SUM($group.amount)"),
|
|
]),
|
|
])
|
|
self.assertViews([
|
|
View(1, sections=[
|
|
Section(1, parentKey="record", tableRef=1, fields=[
|
|
Field(2, colRef=12),
|
|
Field(3, colRef=13),
|
|
]),
|
|
Section(4, parentKey="record", tableRef=2, fields=[
|
|
Field(10, colRef=15),
|
|
Field(12, colRef=17),
|
|
]),
|
|
Section(6, parentKey="record", tableRef=3, fields=[
|
|
Field(16, colRef=18),
|
|
Field(17, colRef=20),
|
|
Field(18, colRef=21),
|
|
]),
|
|
]),
|
|
View(2, sections=[
|
|
Section(2, parentKey="record", tableRef=2, fields=[
|
|
Field(4, colRef=15),
|
|
Field(6, colRef=17),
|
|
]),
|
|
])
|
|
])
|
|
|
|
#----------------------------------------------------------------------
|
|
|
|
@test_engine.test_undo
|
|
def test_summary_column_removals(self):
|
|
# Verify that when we remove a column used for summary-table group-by, it updates summary
|
|
# tables appropriately.
|
|
|
|
self.init_sample_data()
|
|
|
|
# Test that we cannot remove group-by columns from summary tables directly.
|
|
with self.assertRaisesRegex(ValueError, "cannot remove .* group-by"):
|
|
self.apply_user_action(["BulkRemoveRecord", '_grist_Tables_column', [20,18]])
|
|
|
|
# Test that group-by columns in summary tables get removed.
|
|
self.apply_user_action(["BulkRemoveRecord", '_grist_Tables_column', [11,12,16]])
|
|
|
|
# Verify the new structure of tables and views.
|
|
self.assertTables([
|
|
Table(1, "Address", primaryViewId=0, summarySourceTable=0, columns=[
|
|
Column(13, "amount", "Numeric", False, "", 0),
|
|
]),
|
|
Table(2, "Table1", 2, 0, columns=[
|
|
Column(14, "manualSort", "ManualSortPos", False, "", 0),
|
|
Column(15, "A", "Text", False, "", 0),
|
|
Column(17, "C", "Any", True, "", 0),
|
|
]),
|
|
# Note that the summary table here switches to a new one, without the deleted group-by.
|
|
Table(4, "Address_summary", 0, 1, columns=[
|
|
Column(23, "count", "Int", True, summarySourceCol=0, formula="len($group)"),
|
|
Column(24, "amount", "Numeric", True, summarySourceCol=0, formula="SUM($group.amount)"),
|
|
Column(22, "group", "RefList:Address", True, summarySourceCol=0,
|
|
formula="table.getSummarySourceGroup(rec)"),
|
|
]),
|
|
])
|
|
self.assertViews([
|
|
View(1, sections=[
|
|
Section(1, parentKey="record", tableRef=1, fields=[
|
|
Field(3, colRef=13),
|
|
]),
|
|
Section(4, parentKey="record", tableRef=2, fields=[
|
|
Field(10, colRef=15),
|
|
Field(12, colRef=17),
|
|
]),
|
|
Section(6, parentKey="record", tableRef=4, fields=[
|
|
Field(17, colRef=23),
|
|
Field(18, colRef=24),
|
|
]),
|
|
]),
|
|
View(2, sections=[
|
|
Section(2, parentKey="record", tableRef=2, fields=[
|
|
Field(4, colRef=15),
|
|
Field(6, colRef=17),
|
|
]),
|
|
])
|
|
])
|
|
|
|
# Verify the data itself.
|
|
self.assertTableData('Address', data=[
|
|
["id", "amount" ],
|
|
[ 21, 1. ],
|
|
[ 22, 2. ],
|
|
[ 23, 3. ],
|
|
[ 24, 4. ],
|
|
[ 25, 5. ],
|
|
[ 26, 6. ],
|
|
[ 27, 7. ],
|
|
[ 28, 8. ],
|
|
[ 29, 9. ],
|
|
[ 30, 10. ],
|
|
[ 31, 11. ],
|
|
])
|
|
self.assertTableData('Table1', data=[
|
|
["id", "A", "C", "manualSort"],
|
|
[ 1, "a", None, 1.0],
|
|
[ 2, "b", None, 2.0],
|
|
[ 3, "c", None, 3.0],
|
|
])
|
|
self.assertTableData("Address_summary", cols="subset", data=[
|
|
[ "id", "count", "amount" ],
|
|
[ 1, 7+1+1+2, 1.+2+6+7+8+10+11+3+4+5+9 ],
|
|
])
|
|
|
|
#----------------------------------------------------------------------
|
|
|
|
@test_engine.test_undo
|
|
def test_column_sort_removals(self):
|
|
# Verify removal of sort spec entries when columns are removed.
|
|
|
|
self.init_sample_data()
|
|
|
|
# Add sortSpecs to ViewSections.
|
|
self.apply_user_action(['BulkUpdateRecord', '_grist_Views_section', [2, 3, 4],
|
|
{'sortColRefs': ['[15, -16]', '[-15, 16, 17]', '[19]']}
|
|
])
|
|
self.assertTableData('_grist_Views_section', cols="subset", rows="subset", data=[
|
|
["id", "sortColRefs" ],
|
|
[2, '[15, -16]' ],
|
|
[3, '[-15, 16, 17]'],
|
|
[4, '[19]' ],
|
|
])
|
|
|
|
# Remove column, and check that the correct sortColRefs items are removed.
|
|
self.apply_user_action(["RemoveRecord", '_grist_Tables_column', 16])
|
|
self.assertTableData('_grist_Views_section', cols="subset", rows="subset", data=[
|
|
["id", "sortColRefs"],
|
|
[2, '[15]' ],
|
|
[3, '[-15, 17]' ],
|
|
[4, '[19]' ],
|
|
])
|
|
|
|
# Update sortColRefs for next test.
|
|
self.apply_user_action(['UpdateRecord', '_grist_Views_section', 3,
|
|
{'sortColRefs': '[-15, -16, 17]'}
|
|
])
|
|
|
|
# Remove multiple columns using BulkUpdateRecord, and check that the sortSpecs are updated.
|
|
self.apply_user_action(["BulkRemoveRecord", '_grist_Tables_column', [15, 17, 19]])
|
|
self.assertTableData('_grist_Views_section', cols="subset", rows="subset", data=[
|
|
["id", "sortColRefs"],
|
|
[2, '[]' ],
|
|
[3, '[-16]' ],
|
|
[4, '[]' ],
|
|
])
|