gristlabs_grist-core/sandbox/grist/test_summary.py

"""
Test of Summary tables. This has many test cases, so to keep files smaller, it's split into two
files: test_summary.py and test_summary2.py.
"""

import logging

import actions
import summary
import test_engine
import testutil
import useractions
from test_engine import Table, Column, View, Section, Field
from useractions import allowed_summary_change

log = logging.getLogger(__name__)


class TestSummary(test_engine.EngineTestCase):
  sample = testutil.parse_test_sample({
    "SCHEMA": [
      [1, "Address", [
        [11, "city",        "Text",       False, "", "City", ""],
        [12, "state",       "Text",       False, "", "State", "WidgetOptions1"],
        [13, "amount",      "Numeric",    False, "", "Amount", "WidgetOptions2"],
      ]]
    ],
    "DATA": {
      "Address": [
        ["id",  "city",     "state", "amount" ],
        [ 21,   "New York", "NY"   , 1.       ],
        [ 22,   "Albany",   "NY"   , 2.       ],
        [ 23,   "Seattle",  "WA"   , 3.       ],
        [ 24,   "Chicago",  "IL"   , 4.       ],
        [ 25,   "Bedford",  "MA"   , 5.       ],
        [ 26,   "New York", "NY"   , 6.       ],
        [ 27,   "Buffalo",  "NY"   , 7.       ],
        [ 28,   "Bedford",  "NY"   , 8.       ],
        [ 29,   "Boston",   "MA"   , 9.       ],
        [ 30,   "Yonkers",  "NY"   , 10.      ],
        [ 31,   "New York", "NY"   , 11.      ],
      ]
    }
  })

  starting_table = Table(1, "Address", primaryViewId=0, summarySourceTable=0, columns=[
    Column(11, "city",  "Text", isFormula=False, formula="", summarySourceCol=0),
    Column(12, "state", "Text", isFormula=False, formula="", summarySourceCol=0),
    Column(13, "amount", "Numeric", isFormula=False, formula="", summarySourceCol=0),
  ])

  starting_table_data = [
    ["id",  "city",     "state", "amount" ],
    [ 21,   "New York", "NY"   , 1        ],
    [ 22,   "Albany",   "NY"   , 2        ],
    [ 23,   "Seattle",  "WA"   , 3        ],
    [ 24,   "Chicago",  "IL"   , 4        ],
    [ 25,   "Bedford",  "MA"   , 5        ],
    [ 26,   "New York", "NY"   , 6        ],
    [ 27,   "Buffalo",  "NY"   , 7        ],
    [ 28,   "Bedford",  "NY"   , 8        ],
    [ 29,   "Boston",   "MA"   , 9        ],
    [ 30,   "Yonkers",  "NY"   , 10       ],
    [ 31,   "New York", "NY"   , 11       ],
  ]

  #----------------------------------------------------------------------

  def test_encode_summary_table_name(self):
    self.assertEqual(summary.encode_summary_table_name("Foo", []), "Foo_summary")
    self.assertEqual(summary.encode_summary_table_name("Foo", ["A"]), "Foo_summary_A")
    self.assertEqual(summary.encode_summary_table_name("Foo", ["A", "B"]), "Foo_summary_A_B")
    self.assertEqual(summary.encode_summary_table_name("Foo", ["B", "A"]), "Foo_summary_A_B")

  #----------------------------------------------------------------------

  @test_engine.test_undo
  def test_create_view_section(self):
    self.load_sample(self.sample)

    # Verify the starting table; there should be no views yet.
    self.assertTables([self.starting_table])
    self.assertViews([])

    # Create a view + section for the initial table.
    self.apply_user_action(["CreateViewSection", 1, 0, "record", None, None])

    # Verify that we got a new view, with one section, and three fields.
    self.assertTables([self.starting_table])
    basic_view = View(1, sections=[
      Section(1, parentKey="record", tableRef=1, fields=[
        Field(1, colRef=11),
        Field(2, colRef=12),
        Field(3, colRef=13),
      ])
    ])
    self.assertViews([basic_view])

    self.assertTableData("Address", self.starting_table_data)

    # Create a "Totals" section, i.e. a summary with no group-by columns.
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [], None])

    # Verify that a new table gets created, and a new view, with a section for that table,
    # and some auto-generated summary fields.
    summary_table1 = Table(2, "Address_summary", primaryViewId=0, summarySourceTable=1,
                           columns=[
      Column(14, "group", "RefList:Address", isFormula=True, summarySourceCol=0,
             formula="table.getSummarySourceGroup(rec)"),
      Column(15, "count", "Int", isFormula=True, summarySourceCol=0,
             formula="len($group)"),
      Column(16, "amount", "Numeric", isFormula=True, summarySourceCol=0,
             formula="SUM($group.amount)"),
    ])
    summary_view1 = View(2, sections=[
      Section(3, parentKey="record", tableRef=2, fields=[
        Field(6, colRef=15),
        Field(7, colRef=16),
      ])
    ])
    self.assertTables([self.starting_table, summary_table1])
    self.assertViews([basic_view, summary_view1])

    # Verify the summarized data.
    self.assertTableData('Address_summary', cols="subset", data=[
      [ "id", "count",  "amount"],
      [ 1,    11,       66.0    ],
    ])

    # Create a summary section, grouped by the "State" column.
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [12], None])

    # Verify that a new table gets created again, a new view, and a section for that table.
    # Note that we also check that summarySourceTable and summarySourceCol fields are correct.
    summary_table2 = Table(3, "Address_summary_state", primaryViewId=0, summarySourceTable=1,
                           columns=[
      Column(17, "state", "Text", isFormula=False, formula="", summarySourceCol=12),
      Column(18, "group", "RefList:Address", isFormula=True, summarySourceCol=0,
             formula="table.getSummarySourceGroup(rec)"),
      Column(19, "count", "Int", isFormula=True, summarySourceCol=0,
             formula="len($group)"),
      Column(20, "amount", "Numeric", isFormula=True, summarySourceCol=0,
             formula="SUM($group.amount)"),
    ])
    summary_view2 = View(3, sections=[
      Section(5, parentKey="record", tableRef=3, fields=[
        Field(11, colRef=17),
        Field(12, colRef=19),
        Field(13, colRef=20),
      ])
    ])
    self.assertTables([self.starting_table, summary_table1, summary_table2])
    self.assertViews([basic_view, summary_view1, summary_view2])

    # Verify more fields of the new column objects.
    self.assertTableData('_grist_Tables_column', rows="subset", cols="subset", data=[
      ['id', 'colId',  'type',    'formula',            'widgetOptions', 'label'],
      [17,   'state',  'Text',    '',                   'WidgetOptions1', 'State'],
      [20,   'amount', 'Numeric', 'SUM($group.amount)', 'WidgetOptions2', 'Amount'],
    ])

    # Verify the summarized data.
    self.assertTableData('Address_summary_state', cols="subset", data=[
      [ "id", "state", "count", "amount"          ],
      [ 1,    "NY",     7,      1.+2+6+7+8+10+11  ],
      [ 2,    "WA",     1,      3.                ],
      [ 3,    "IL",     1,      4.                ],
      [ 4,    "MA",     2,      5.+9              ],
    ])

    # Create a summary section grouped by two columns ("city" and "state").
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])

    # Verify the new table and views.
    summary_table3 = Table(4, "Address_summary_city_state", primaryViewId=0, summarySourceTable=1,
                           columns=[
      Column(21, "city", "Text", isFormula=False, formula="", summarySourceCol=11),
      Column(22, "state", "Text", isFormula=False, formula="", summarySourceCol=12),
      Column(23, "group", "RefList:Address", isFormula=True, summarySourceCol=0,
             formula="table.getSummarySourceGroup(rec)"),
      Column(24, "count", "Int", isFormula=True, summarySourceCol=0,
             formula="len($group)"),
      Column(25, "amount", "Numeric", isFormula=True, summarySourceCol=0,
             formula="SUM($group.amount)"),
    ])
    summary_view3 = View(4, sections=[
      Section(7, parentKey="record", tableRef=4, fields=[
        Field(18, colRef=21),
        Field(19, colRef=22),
        Field(20, colRef=24),
        Field(21, colRef=25),
      ])
    ])
    self.assertTables([self.starting_table, summary_table1, summary_table2, summary_table3])
    self.assertViews([basic_view, summary_view1, summary_view2, summary_view3])

    # Verify the summarized data.
    self.assertTableData('Address_summary_city_state', cols="subset", data=[
      [ "id", "city",     "state", "count", "amount"  ],
      [ 1,    "New York", "NY"   , 3,       1.+6+11   ],
      [ 2,    "Albany",   "NY"   , 1,       2.        ],
      [ 3,    "Seattle",  "WA"   , 1,       3.        ],
      [ 4,    "Chicago",  "IL"   , 1,       4.        ],
      [ 5,    "Bedford",  "MA"   , 1,       5.        ],
      [ 6,    "Buffalo",  "NY"   , 1,       7.        ],
      [ 7,    "Bedford",  "NY"   , 1,       8.        ],
      [ 8,    "Boston",   "MA"   , 1,       9.        ],
      [ 9,    "Yonkers",  "NY"   , 1,       10.       ],
    ])

    # The original table's data should not have changed.
    self.assertTableData("Address", self.starting_table_data)

  #----------------------------------------------------------------------

  def test_summary_gencode(self):
    self.maxDiff = 1000       # If there is a discrepancy, allow the bigger diff.
    self.load_sample(self.sample)
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [], None])
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])
    self.assertMultiLineEqual(self.engine.fetch_table_schema(),
"""import grist
from functions import *       # global uppercase functions
import datetime, math, re     # modules commonly needed in formulas


@grist.UserTable
class Address:
  city = grist.Text()
  state = grist.Text()
  amount = grist.Numeric()

  class _Summary:

    @grist.formulaType(grist.ReferenceList('Address'))
    def group(rec, table):
      return table.getSummarySourceGroup(rec)

    @grist.formulaType(grist.Int())
    def count(rec, table):
      return len(rec.group)

    @grist.formulaType(grist.Numeric())
    def amount(rec, table):
      return SUM(rec.group.amount)
""")

  #----------------------------------------------------------------------

  @test_engine.test_undo
  def test_summary_table_reuse(self):
    # Test that we'll reuse a suitable summary table when already available.

    self.load_sample(self.sample)

    # Create a summary section grouped by two columns ("city" and "state").
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])

    # Verify the new table and views.
    summary_table = Table(2, "Address_summary_city_state", primaryViewId=0, summarySourceTable=1,
                           columns=[
      Column(14, "city", "Text", isFormula=False, formula="", summarySourceCol=11),
      Column(15, "state", "Text", isFormula=False, formula="", summarySourceCol=12),
      Column(16, "group", "RefList:Address", isFormula=True, summarySourceCol=0,
             formula="table.getSummarySourceGroup(rec)"),
      Column(17, "count", "Int", isFormula=True, summarySourceCol=0,
             formula="len($group)"),
      Column(18, "amount", "Numeric", isFormula=True, summarySourceCol=0,
             formula="SUM($group.amount)"),
    ])
    summary_view = View(1, sections=[
      Section(2, parentKey="record", tableRef=2, fields=[
        Field(5, colRef=14),
        Field(6, colRef=15),
        Field(7, colRef=17),
        Field(8, colRef=18),
      ])
    ])
    self.assertTables([self.starting_table, summary_table])
    self.assertViews([summary_view])

    # Create twoo other views + view sections with the same breakdown (in different order
    # of group-by fields, which should still reuse the same table).
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [12,11], None])
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])
    summary_view2 = View(2, sections=[
      Section(3, parentKey="record", tableRef=2, fields=[
        Field(9, colRef=15),
        Field(10, colRef=14),
        Field(11, colRef=17),
        Field(12, colRef=18),
      ])
    ])
    summary_view3 = View(3, sections=[
      Section(4, parentKey="record", tableRef=2, fields=[
        Field(13, colRef=14),
        Field(14, colRef=15),
        Field(15, colRef=17),
        Field(16, colRef=18),
      ])
    ])
    # Verify that we have a new view, but are reusing the table.
    self.assertTables([self.starting_table, summary_table])
    self.assertViews([summary_view, summary_view2, summary_view3])

    # Verify the summarized data.
    self.assertTableData('Address_summary_city_state', cols="subset", data=[
      [ "id", "city",     "state", "count", "amount"  ],
      [ 1,    "New York", "NY"   , 3,       1.+6+11   ],
      [ 2,    "Albany",   "NY"   , 1,       2.        ],
      [ 3,    "Seattle",  "WA"   , 1,       3.        ],
      [ 4,    "Chicago",  "IL"   , 1,       4.        ],
      [ 5,    "Bedford",  "MA"   , 1,       5.        ],
      [ 6,    "Buffalo",  "NY"   , 1,       7.        ],
      [ 7,    "Bedford",  "NY"   , 1,       8.        ],
      [ 8,    "Boston",   "MA"   , 1,       9.        ],
      [ 9,    "Yonkers",  "NY"   , 1,       10.       ],
    ])

  #----------------------------------------------------------------------

  @test_engine.test_undo
  def test_summary_no_invalid_reuse(self):
    # Verify that if we have some summary tables for one table, they don't mistakenly get used
    # when we need a summary for another table.

    # Load table and create a couple summary sections, for totals, and grouped by "state".
    self.load_sample(self.sample)
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [], None])
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [12], None])

    self.assertTables([
      self.starting_table,
      Table(2, "Address_summary", 0, 1, columns=[
        Column(14, "group",   "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
        Column(15, "count",   "Int",      True,   "len($group)", 0),
        Column(16, "amount",  "Numeric",  True,   "SUM($group.amount)", 0),
      ]),
      Table(3, "Address_summary_state", 0, 1, columns=[
        Column(17, "state",   "Text",     False,  "", 12),
        Column(18, "group",   "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
        Column(19, "count",   "Int",      True,   "len($group)", 0),
        Column(20, "amount",  "Numeric",  True,   "SUM($group.amount)", 0),
      ]),
    ])

    # Create another table similar to the first one.
    self.apply_user_action(["AddTable", "Address2", [
      { "id": "city", "type": "Text" },
      { "id": "state", "type": "Text" },
      { "id": "amount", "type": "Numeric" },
    ]])
    data = self.sample["DATA"]["Address"]
    self.apply_user_action(["BulkAddRecord", "Address2", data.row_ids, data.columns])

    # Check that we've loaded the right data, and have the new table.
    self.assertTableData("Address", cols="subset", data=self.starting_table_data)
    self.assertTableData("Address2", cols="subset", data=self.starting_table_data)
    self.assertTableData("_grist_Tables", cols="subset", data=[
      ['id',    'tableId',  'summarySourceTable'],
      [ 1,      'Address',                  0],
      [ 2,      'Address_summary',   1],
      [ 3,      'Address_summary_state',  1],
      [ 4,      'Address2',                 0],
    ])

    # Now create similar summary sections for the new table.
    self.apply_user_action(["CreateViewSection", 4, 0, "record", [], None])
    self.apply_user_action(["CreateViewSection", 4, 0, "record", [23], None])

    # Make sure this creates new section rather than reuses similar ones for the wrong table.
    self.assertTables([
      self.starting_table,
      Table(2, "Address_summary", 0, 1, columns=[
        Column(14, "group",   "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
        Column(15, "count",   "Int",      True,   "len($group)", 0),
        Column(16, "amount",  "Numeric",  True,   "SUM($group.amount)", 0),
      ]),
      Table(3, "Address_summary_state", 0, 1, columns=[
        Column(17, "state",   "Text",     False,  "", 12),
        Column(18, "group",   "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
        Column(19, "count",   "Int",      True,   "len($group)", 0),
        Column(20, "amount",  "Numeric",  True,   "SUM($group.amount)", 0),
      ]),
      Table(4, "Address2", primaryViewId=3, summarySourceTable=0, columns=[
        Column(21, "manualSort", "ManualSortPos",False, "", 0),
        Column(22, "city",    "Text",     False, "", 0),
        Column(23, "state",   "Text",     False, "", 0),
        Column(24, "amount",  "Numeric",  False, "", 0),
      ]),
      Table(5, "Address2_summary", 0, 4, columns=[
        Column(25, "group",   "RefList:Address2", True, "table.getSummarySourceGroup(rec)", 0),
        Column(26, "count",   "Int",      True,   "len($group)", 0),
        Column(27, "amount",  "Numeric",  True,   "SUM($group.amount)", 0),
      ]),
      Table(6, "Address2_summary_state", 0, 4, columns=[
        Column(28, "state",   "Text",     False,  "", 23),
        Column(29, "group",   "RefList:Address2", True, "table.getSummarySourceGroup(rec)", 0),
        Column(30, "count",   "Int",      True,   "len($group)", 0),
        Column(31, "amount",  "Numeric",  True,   "SUM($group.amount)", 0),
      ]),
    ])

  #----------------------------------------------------------------------

  @test_engine.test_undo
  def test_summary_updates(self):
    # Verify that summary tables update automatically when we change a value used in a summary
    # formula; or a value in a group-by column; or add/remove a record; that records get
    # auto-added when new group-by combinations appear.

    # Load sample and create a summary section grouped by two columns ("city" and "state").
    self.load_sample(self.sample)
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])

    # Verify that the summary table respects all updates to the source table.
    self._do_test_updates("Address", "Address_summary_city_state")

  def _do_test_updates(self, source_tbl_name, summary_tbl_name):
    # This is the main part of test_summary_updates(). It's moved to its own method so that
    # updates can be verified the same way after a table rename.

    # Verify the summarized data.
    self.assertTableData(summary_tbl_name, cols="subset", data=[
      [ "id", "city",     "state", "count", "amount"  ],
      [ 1,    "New York", "NY"   , 3,       1.+6+11   ],
      [ 2,    "Albany",   "NY"   , 1,       2.        ],
      [ 3,    "Seattle",  "WA"   , 1,       3.        ],
      [ 4,    "Chicago",  "IL"   , 1,       4.        ],
      [ 5,    "Bedford",  "MA"   , 1,       5.        ],
      [ 6,    "Buffalo",  "NY"   , 1,       7.        ],
      [ 7,    "Bedford",  "NY"   , 1,       8.        ],
      [ 8,    "Boston",   "MA"   , 1,       9.        ],
      [ 9,    "Yonkers",  "NY"   , 1,       10.       ],
    ])

    # Change an amount (New York, NY, 6 -> 106), check that the right calc action gets emitted.
    out_actions = self.update_record(source_tbl_name, 26, amount=106)
    self.assertPartialOutActions(out_actions, {
      "stored": [
        actions.UpdateRecord(source_tbl_name, 26, {'amount': 106}),
        actions.UpdateRecord(summary_tbl_name, 1, {'amount': 1.+106+11}),
      ]
    })

    # Change a groupby value so that a record moves from one summary group to another.
    # Bedford, NY, 8.0 -> Bedford, MA, 8.0
    out_actions = self.update_record(source_tbl_name, 28, state="MA")
    self.assertPartialOutActions(out_actions, {
      "stored": [
        actions.UpdateRecord(source_tbl_name, 28, {'state': 'MA'}),
        actions.RemoveRecord(summary_tbl_name, 7),
        actions.UpdateRecord(summary_tbl_name, 5, {'amount': 5.0 + 8.0}),
        actions.UpdateRecord(summary_tbl_name, 5, {'count': 2}),
        actions.UpdateRecord(summary_tbl_name, 5, {'group': [25, 28]}),
      ]
    })

    # Add a record to an existing group (Bedford, MA, 108.0)
    out_actions = self.add_record(source_tbl_name, city="Bedford", state="MA", amount=108.0)
    self.assertPartialOutActions(out_actions, {
      "stored": [
        actions.AddRecord(source_tbl_name, 32,
                          {'city': 'Bedford', 'state': 'MA', 'amount': 108.0}),
        actions.UpdateRecord(summary_tbl_name, 5, {'amount': 5.0 + 8.0 + 108.0}),
        actions.UpdateRecord(summary_tbl_name, 5, {'count': 3}),
        actions.UpdateRecord(summary_tbl_name, 5, {'group': [25, 28, 32]}),
      ]
    })

    # Remove a record (rowId=28, Bedford, MA, 8.0)
    out_actions = self.remove_record(source_tbl_name, 28)
    self.assertPartialOutActions(out_actions, {
      "stored": [
        actions.RemoveRecord(source_tbl_name, 28),
        actions.UpdateRecord(summary_tbl_name, 5, {'amount': 5.0 + 108.0}),
        actions.UpdateRecord(summary_tbl_name, 5, {'count': 2}),
        actions.UpdateRecord(summary_tbl_name, 5, {'group': [25, 32]}),
      ]
    })

    # Change groupby value to create a new combination (rowId 25, Bedford, MA, 5.0 -> Salem, MA).
    # A new summary record should be added.
    out_actions = self.update_record(source_tbl_name, 25, city="Salem")
    self.assertPartialOutActions(out_actions, {
      "stored": [
        actions.UpdateRecord(source_tbl_name, 25, {'city': 'Salem'}),
        actions.AddRecord(summary_tbl_name, 10, {'city': 'Salem', 'state': 'MA'}),
        actions.BulkUpdateRecord(summary_tbl_name, [5,10], {'amount': [108.0, 5.0]}),
        actions.BulkUpdateRecord(summary_tbl_name, [5,10], {'count': [1, 1]}),
        actions.BulkUpdateRecord(summary_tbl_name, [5,10], {'group': [[32], [25]]}),
      ]
    })

    # Add a record with a new combination (Amherst, MA, 17)
    out_actions = self.add_record(source_tbl_name, city="Amherst", state="MA", amount=17.0)
    self.assertPartialOutActions(out_actions, {
      "stored": [
        actions.AddRecord(source_tbl_name, 33, {'city': 'Amherst', 'state': 'MA', 'amount': 17.}),
        actions.AddRecord(summary_tbl_name, 11, {'city': 'Amherst', 'state': 'MA'}),
        actions.UpdateRecord(summary_tbl_name, 11, {'amount': 17.0}),
        actions.UpdateRecord(summary_tbl_name, 11, {'count': 1}),
        actions.UpdateRecord(summary_tbl_name, 11, {'group': [33]}),
      ]
    })

    # Add a new source record which creates a new summary row,
    # then delete the source row which implicitly deletes the summary row.
    # Overall this is a no-op, but it tests a specific undo-related bugfix.
    self.add_record(source_tbl_name, city="Nowhere", state="??", amount=666)
    out_actions = self.remove_record(source_tbl_name, 34)
    self.assertOutActions(out_actions, {
      'calc': [],
      'direct': [True, False],
      'stored': [
        ['RemoveRecord', source_tbl_name, 34],
        ['RemoveRecord', summary_tbl_name, 12],
      ],
      'undo': [
        ['UpdateRecord', summary_tbl_name, 12, {'group': ['L', 34]}],
        ['UpdateRecord', summary_tbl_name, 12, {'count': 1}],
        ['UpdateRecord', summary_tbl_name, 12, {'amount': 666.0}],
        ['AddRecord', source_tbl_name, 34, {'amount': 666.0, 'city': 'Nowhere', 'state': '??'}],
        ['AddRecord', summary_tbl_name, 12,
         {'city': 'Nowhere', 'group': ['L'], 'state': '??'}],
      ],
    })

    # Verify the resulting data after all the updates.
    self.assertTableData(summary_tbl_name, cols="subset", data=[
      [ "id", "city",     "state", "count", "amount"  ],
      [ 1,    "New York", "NY"   , 3,       1.+106+11 ],
      [ 2,    "Albany",   "NY"   , 1,       2.        ],
      [ 3,    "Seattle",  "WA"   , 1,       3.        ],
      [ 4,    "Chicago",  "IL"   , 1,       4.        ],
      [ 5,    "Bedford",  "MA"   , 1,       108.      ],
      [ 6,    "Buffalo",  "NY"   , 1,       7.        ],
      [ 8,    "Boston",   "MA"   , 1,       9.        ],
      [ 9,    "Yonkers",  "NY"   , 1,       10.       ],
      [ 10,   "Salem",    "MA"   , 1,       5.0       ],
      [ 11,   "Amherst",  "MA"   , 1,       17.0      ],
    ])

  #----------------------------------------------------------------------

  @test_engine.test_undo
  def test_table_rename(self):
    # Verify that summary tables keep working and updating when source table is renamed.

    # Load sample and create a couple of summary sections.
    self.load_sample(self.sample)
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])

    # Check what tables we have now.
    self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
      [1, "Address",                  0],
      [2, "Address_summary_city_state",   1],
    ])

    # Add a column to the summary table with a name that doesn't exist in the source table,
    # to test a specific bug fix.
    self.add_column(
      "Address_summary_city_state", "lookup",
      formula="Address.lookupRecords(city=$city)", isFormula=True
    )

    # Rename the table: this is what we are really testing in this test case.
    self.apply_user_action(["RenameTable", "Address", "Location"])

    self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
      [1, "Location",                  0],
      [2, "Location_summary_city_state",   1],
    ])

    # Check that the summary table column's formula was updated correctly.
    self.assertTableData("_grist_Tables_column", cols="subset", rows="subset", data=[
      ["id", "colId", "formula"],
      [19, "lookup", "Location.lookupRecords(city=$city)"],
    ])
    # This column isn't expected in _do_test_updates().
    self.remove_column("Location_summary_city_state", "lookup")

    # Verify that the bigger summary table respects all updates to the renamed source table.
    self._do_test_updates("Location", "Location_summary_city_state")

  #----------------------------------------------------------------------

  @test_engine.test_undo
  def test_table_rename_multiple(self):
    # Similar to the above, verify renames, but now with two summary tables.

    self.load_sample(self.sample)
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [], None])
    self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
      [1, "Address",                  0],
      [2, "Address_summary_city_state",   1],
      [3, "Address_summary",  1],
    ])
    # Verify the data in the simple totals-only summary table.
    self.assertTableData('Address_summary', cols="subset", data=[
      [ "id", "count",  "amount"],
      [ 1,    11,       66.0    ],
    ])

    # Do a rename.
    self.apply_user_action(["RenameTable", "Address", "Addresses"])
    self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
      [1, "Addresses",                  0],
      [2, "Addresses_summary_city_state",   1],
      [3, "Addresses_summary",  1],
    ])
    self.assertTableData('Addresses_summary', cols="subset", data=[
      [ "id", "count",  "amount"],
      [ 1,    11,       66.0    ],
    ])

    # Remove one of the tables so that we can use _do_test_updates to verify updates still work.
    self.apply_user_action(["RemoveTable", "Addresses_summary"])
    self.assertPartialData("_grist_Tables", ["id", "tableId", "summarySourceTable"], [
      [1, "Addresses",                  0],
      [2, "Addresses_summary_city_state",   1],
    ])
    self._do_test_updates("Addresses", "Addresses_summary_city_state")

  #----------------------------------------------------------------------

  @test_engine.test_undo
  def test_change_summary_formula(self):
    # Verify that changing a summary formula affects all group-by variants, and adding a new
    # summary table gets the changed formula.
    #
    # (Recall that all summaries of a single table are *conceptually* variants of a single summary
    # table, sharing all formulas and differing only in the group-by columns.)

    self.load_sample(self.sample)
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [11,12], None])
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [], None])

    # These are the tables and columns we automatically get.
    self.assertTables([
      self.starting_table,
      Table(2, "Address_summary_city_state", 0, 1, columns=[
        Column(14, "city",    "Text",     False,  "", 11),
        Column(15, "state",   "Text",     False,  "", 12),
        Column(16, "group",   "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
        Column(17, "count",   "Int",      True,   "len($group)", 0),
        Column(18, "amount",  "Numeric",  True,   "SUM($group.amount)", 0),
      ]),
      Table(3, "Address_summary", 0, 1, columns=[
        Column(19, "group",   "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
        Column(20, "count",   "Int",      True,   "len($group)", 0),
        Column(21, "amount",  "Numeric",  True,   "SUM($group.amount)", 0),
      ])
    ])

    # Now change a formula using one of the summary tables. It should trigger an equivalent
    # change in the other.
    self.apply_user_action(["ModifyColumn", "Address_summary_city_state", "amount",
                            {"formula": "10*sum($group.amount)"}])
    self.assertTableData('_grist_Tables_column', rows="subset", cols="subset", data=[
      ['id', 'colId',  'type',    'formula',               'widgetOptions', 'label'],
      [18,   'amount', 'Numeric', '10*sum($group.amount)', 'WidgetOptions2', 'Amount'],
      [21,   'amount', 'Numeric', '10*sum($group.amount)', 'WidgetOptions2', 'Amount'],
    ])

    # Change a formula and a few other fields in the other table, and verify a change to both.
    self.apply_user_action(["ModifyColumn", "Address_summary", "amount",
                            {"formula": "100*sum($group.amount)",
                             "type": "Text",
                             "widgetOptions": "hello",
                             "label": "AMOUNT",
                             "untieColIdFromLabel": True
                            }])
    self.assertTableData('_grist_Tables_column', rows="subset", cols="subset", data=[
      ['id', 'colId',  'type', 'formula',                 'widgetOptions', 'label'],
      [18,   'amount', 'Text', '100*sum($group.amount)',  'hello', 'AMOUNT'],
      [21,   'amount', 'Text', '100*sum($group.amount)',  'hello', 'AMOUNT'],
    ])

    # Check the values in the summary tables: they should reflect the new formula.
    self.assertTableData('Address_summary_city_state', cols="subset", data=[
      [ "id", "city",     "state", "count", "amount"  ],
      [ 1,    "New York", "NY"   , 3,       str(100*(1+6+11))],
      [ 2,    "Albany",   "NY"   , 1,       "200"        ],
      [ 3,    "Seattle",  "WA"   , 1,       "300"        ],
      [ 4,    "Chicago",  "IL"   , 1,       "400"        ],
      [ 5,    "Bedford",  "MA"   , 1,       "500"        ],
      [ 6,    "Buffalo",  "NY"   , 1,       "700"        ],
      [ 7,    "Bedford",  "NY"   , 1,       "800"        ],
      [ 8,    "Boston",   "MA"   , 1,       "900"        ],
      [ 9,    "Yonkers",  "NY"   , 1,       "1000"       ],
    ])
    self.assertTableData('Address_summary', cols="subset", data=[
      [ "id", "count",  "amount"],
      [ 1,    11,       "6600"],
    ])

    # Add a new summary table, and check that it gets the new formula.
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [12], None])
    self.assertTables([
      self.starting_table,
      Table(2, "Address_summary_city_state", 0, 1, columns=[
        Column(14, "city",    "Text",     False,  "", 11),
        Column(15, "state",   "Text",     False,  "", 12),
        Column(16, "group",   "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
        Column(17, "count",   "Int",      True,   "len($group)", 0),
        Column(18, "amount",  "Text",     True,   "100*sum($group.amount)", 0),
      ]),
      Table(3, "Address_summary", 0, 1, columns=[
        Column(19, "group",   "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
        Column(20, "count",   "Int",      True,   "len($group)", 0),
        Column(21, "amount",  "Text",     True,   "100*sum($group.amount)", 0),
      ]),
      Table(4, "Address_summary_state", 0, 1, columns=[
        Column(22, "state",   "Text",     False,  "", 12),
        Column(23, "group",   "RefList:Address", True, "table.getSummarySourceGroup(rec)", 0),
        Column(24, "count",   "Int",      True,   "len($group)", 0),
        Column(25, "amount",  "Text",     True,   "100*sum($group.amount)", 0),
      ])
    ])
    self.assertTableData('_grist_Tables_column', rows="subset", cols="subset", data=[
      ['id', 'colId',  'type', 'formula',                 'widgetOptions', 'label'],
      [18,   'amount', 'Text', '100*sum($group.amount)',  'hello', 'AMOUNT'],
      [21,   'amount', 'Text', '100*sum($group.amount)',  'hello', 'AMOUNT'],
      [25,   'amount', 'Text', '100*sum($group.amount)',  'hello', 'AMOUNT'],
    ])

    # Verify the summarized data.
    self.assertTableData('Address_summary_state', cols="subset", data=[
      [ "id", "state", "count", "amount"                    ],
      [ 1,    "NY",     7,      str(int(100*(1.+2+6+7+8+10+11))) ],
      [ 2,    "WA",     1,      "300"                     ],
      [ 3,    "IL",     1,      "400"                     ],
      [ 4,    "MA",     2,      str(500+900)               ],
    ])

  #----------------------------------------------------------------------
  @test_engine.test_undo
  def test_convert_source_column(self):
    # Verify that we can convert the type of a column when there is a summary table using that
    # column to group by. Since converting generates extra summary records, this may cause bugs.

    self.apply_user_action(["AddEmptyTable", None])
    self.apply_user_action(["BulkAddRecord", "Table1", [None]*3, {"A": [10,20,10], "B": [1,2,3]}])
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [2], None])

    # Verify metadata and actual data initially.
    self.assertTables([
      Table(1, "Table1", summarySourceTable=0, primaryViewId=1, columns=[
        Column(1, "manualSort", "ManualSortPos",  False,  "", 0),
        Column(2, "A",          "Numeric",        False,  "", 0),
        Column(3, "B",          "Numeric",        False,  "", 0),
        Column(4, "C",          "Any",            True,   "", 0),
      ]),
      Table(2, "Table1_summary_A", summarySourceTable=1, primaryViewId=0, columns=[
        Column(5, "A",          "Numeric",        False,  "", 2),
        Column(6, "group",      "RefList:Table1", True,  "table.getSummarySourceGroup(rec)", 0),
        Column(7, "count",      "Int",            True,  "len($group)", 0),
        Column(8, "B",          "Numeric",        True,  "SUM($group.B)", 0),
      ])
    ])
    self.assertTableData('Table1', data=[
      [ "id", "manualSort", "A",  "B",  "C"   ],
      [ 1,    1.0,          10,   1.0,    None  ],
      [ 2,    2.0,          20,   2.0,    None  ],
      [ 3,    3.0,          10,   3.0,    None  ],
    ])
    self.assertTableData('Table1_summary_A', data=[
      [ "id", "A",  "group",  "count",  "B" ],
      [ 1,    10,   [1,3],    2,        4   ],
      [ 2,    20,   [2],      1,        2   ],
    ])


    # Do a conversion.
    self.apply_user_action(["UpdateRecord", "_grist_Tables_column", 2, {"type": "Text"}])

    # Verify that the conversion's result is as expected.
    self.assertTables([
      Table(1, "Table1", summarySourceTable=0, primaryViewId=1, columns=[
        Column(1, "manualSort", "ManualSortPos",  False,  "", 0),
        Column(2, "A",          "Text",           False,  "", 0),
        Column(3, "B",          "Numeric",        False,  "", 0),
        Column(4, "C",          "Any",            True,   "", 0),
      ]),
      Table(2, "Table1_summary_A", summarySourceTable=1, primaryViewId=0, columns=[
        Column(5, "A",          "Text",           False,  "", 2),
        Column(6, "group",      "RefList:Table1", True,  "table.getSummarySourceGroup(rec)", 0),
        Column(7, "count",      "Int",            True,  "len($group)", 0),
        Column(8, "B",          "Numeric",        True,  "SUM($group.B)", 0),
      ])
    ])
    self.assertTableData('Table1', data=[
      [ "id", "manualSort", "A",  "B",  "C"   ],
      [ 1,    1.0,          "10", 1.0,  None  ],
      [ 2,    2.0,          "20", 2.0,  None  ],
      [ 3,    3.0,          "10", 3.0,  None  ],
    ])
    self.assertTableData('Table1_summary_A', data=[
      [ "id", "A",  "group",  "count",  "B" ],
      [ 1,    "10", [1,3],    2,        4   ],
      [ 2,    "20", [2],      1,        2   ],
    ])

  #----------------------------------------------------------------------
  @test_engine.test_undo
  def test_remove_source_column(self):
    # Verify that we can remove a column when there is a summary table using that column to group
    # by. (Bug T188.)

    self.apply_user_action(["AddEmptyTable", None])
    self.apply_user_action(["BulkAddRecord", "Table1", [None]*3,
                            {"A": ['a','b','c'], "B": [1,1,2], "C": [4,5,6]}])
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [2,3], None])

    # Verify metadata and actual data initially.
    self.assertTables([
      Table(1, "Table1", summarySourceTable=0, primaryViewId=1, columns=[
        Column(1, "manualSort", "ManualSortPos",  False,  "", 0),
        Column(2, "A",          "Text",           False,  "", 0),
        Column(3, "B",          "Numeric",        False,  "", 0),
        Column(4, "C",          "Numeric",        False,  "", 0),
      ]),
      Table(2, "Table1_summary_A_B", summarySourceTable=1, primaryViewId=0, columns=[
        Column(5, "A",          "Text",           False,  "", 2),
        Column(6, "B",          "Numeric",        False,  "", 3),
        Column(7, "group",      "RefList:Table1", True,  "table.getSummarySourceGroup(rec)", 0),
        Column(8, "count",      "Int",            True,  "len($group)", 0),
        Column(9, "C",          "Numeric",        True,  "SUM($group.C)", 0),
      ])
    ])
    self.assertTableData('Table1', data=[
      [ "id", "manualSort", "A",  "B",  "C" ],
      [ 1,    1.0,          'a',  1.0,  4   ],
      [ 2,    2.0,          'b',  1.0,  5   ],
      [ 3,    3.0,          'c',  2.0,  6   ],
    ])
    self.assertTableData('Table1_summary_A_B', data=[
      [ "id", "A",  "B",  "group",  "count",  "C" ],
      [ 1,    'a',  1.0,  [1],      1,        4   ],
      [ 2,    'b',  1.0,  [2],      1,        5   ],
      [ 3,    'c',  2.0,  [3],      1,        6   ],
    ])

    # Remove column A, used for group-by.
    self.apply_user_action(["RemoveColumn", "Table1", "A"])

    # Verify that the conversion's result is as expected.
    self.assertTables([
      Table(1, "Table1", summarySourceTable=0, primaryViewId=1, columns=[
        Column(1, "manualSort", "ManualSortPos",  False,  "", 0),
        Column(3, "B",          "Numeric",        False,  "", 0),
        Column(4, "C",          "Numeric",        False,  "", 0),
      ]),
      Table(3, "Table1_summary_B", summarySourceTable=1, primaryViewId=0, columns=[
        Column(10, "B",          "Numeric",        False,  "", 3),
        Column(12, "count",      "Int",            True,  "len($group)", 0),
        Column(13, "C",          "Numeric",        True,  "SUM($group.C)", 0),
        Column(11, "group",      "RefList:Table1", True,  "table.getSummarySourceGroup(rec)", 0),
      ])
    ])
    self.assertTableData('Table1', data=[
      [ "id", "manualSort", "B",  "C" ],
      [ 1,    1.0,          1.0,  4   ],
      [ 2,    2.0,          1.0,  5   ],
      [ 3,    3.0,          2.0,  6   ],
    ])
    self.assertTableData('Table1_summary_B', data=[
      [ "id", "B",  "group",  "count",  "C" ],
      [ 1,    1.0,  [1,2],    2,        9   ],
      [ 2,    2.0,  [3],      1,        6   ],
    ])

  def test_remove_source_columns_with_display_column(self):
    # Verify a fix for a specific bug: removing multiple groupby source columns
    # when the summary table contains a display column.

    self.apply_user_action(["AddEmptyTable", None])
    # Group by A and B
    self.apply_user_action(["CreateViewSection", 1, 0, "record", [2,3], None])
    # Add a display column for the group column
    self.apply_user_action(['SetDisplayFormula', 'Table1_summary_A_B', None, 7, '$group.C'])

    # Verify metadata and initially.
    self.assertTables([
      Table(1, "Table1", summarySourceTable=0, primaryViewId=1, columns=[
        Column(1, "manualSort", "ManualSortPos",  False,  "", 0),
        Column(2, "A",          "Any",        True,  "", 0),
        Column(3, "B",          "Any",        True,  "", 0),
        Column(4, "C",          "Any",        True,  "", 0),
      ]),
      Table(2, "Table1_summary_A_B", summarySourceTable=1, primaryViewId=0, columns=[
        Column(5, "A",          "Any",            False, "", 2),
        Column(6, "B",          "Any",            False, "", 3),
        Column(7, "group",      "RefList:Table1", True,  "table.getSummarySourceGroup(rec)", 0),
        Column(8, "count",      "Int",            True,  "len($group)", 0),
        Column(9, "gristHelper_Display", "Any",   True,  "$group.C", 0),
      ])
    ])

    user_actions = [
      useractions.from_repr(ua) for ua in
      [
        ['RemoveColumn', 'Table1', 'A'],
        ['RemoveColumn', 'Table1', 'B'],
      ]
    ]
    self.engine.apply_user_actions(user_actions)

    # Verify that the final structure is as expected.
    self.assertTables([
      Table(1, "Table1", summarySourceTable=0, primaryViewId=1, columns=[
        Column(1, "manualSort", "ManualSortPos",  False,  "", 0),
        Column(4, "C",          "Any",        True,  "", 0),
      ]),
      # Table1_summary_A_B was removed and recreated as Table1_summary.
      # This removed Table1_summary_A_B.group which automatically removed gristHelper_Display
      # which led to an error in the past.
      Table(4, "Table1_summary", summarySourceTable=1, primaryViewId=0, columns=[
        Column(14, "group",      "RefList:Table1", True,  "table.getSummarySourceGroup(rec)", 0),
        Column(15, "count",      "Int",            True,  "len($group)", 0),
      ])
    ])

  #----------------------------------------------------------------------
  # pylint: disable=R0915
  def test_allow_select_by_change(self):
    def widgetOptions(n, o):
      return allowed_summary_change('widgetOptions', n, o)

    # Can make no update on widgetOptions.
    new = None
    old = None
    self.assertTrue(widgetOptions(new, old))

    new = ''
    old = None
    self.assertTrue(widgetOptions(new, old))

    new = ''
    old = ''
    self.assertTrue(widgetOptions(new, old))

    new = None
    old = ''
    self.assertTrue(widgetOptions(new, old))

    # Can update when key was not present
    new = '{"widget":"TextBox","alignment":"center"}'
    old = ''
    self.assertTrue(widgetOptions(new, old))

    new = ''
    old = '{"widget":"TextBox","alignment":"center"}'
    self.assertTrue(widgetOptions(new, old))

    # Can update when key was present.
    new = '{"widget":"TextBox","alignment":"center"}'
    old = '{"widget":"Spinner","alignment":"center"}'
    self.assertTrue(widgetOptions(new, old))

    # Can update but must leave other options.
    new = '{"widget":"TextBox","cant":"center"}'
    old = '{"widget":"Spinner","cant":"center"}'
    self.assertTrue(widgetOptions(new, old))

    # Can't add protected property when old was empty.
    new = '{"widget":"TextBox","cant":"new"}'
    old = None
    self.assertFalse(widgetOptions(new, old))

    # Can't remove when there was a protected property.
    new = None
    old = '{"widget":"TextBox","cant":"old"}'
    self.assertFalse(widgetOptions(new, old))

    # Can't update by omitting.
    new = '{"widget":"TextBox"}'
    old = '{"widget":"TextBox","cant":"old"}'
    self.assertFalse(widgetOptions(new, old))

    # Can't update by changing.
    new = '{"widget":"TextBox","cant":"new"}'
    old = '{"widget":"TextBox","cant":"old"}'
    self.assertFalse(widgetOptions(new, old))

    # Can't update by adding.
    new = '{"widget":"TextBox","cant":"new"}'
    old = '{"widget":"TextBox"}'
    self.assertFalse(widgetOptions(new, old))

    # Can update objects
    new = '{"widget":"TextBox","alignment":{"prop":1},"cant":{"prop":1}}'
    old = '{"widget":"TextBox","alignment":{"prop":2},"cant":{"prop":1}}'
    self.assertTrue(widgetOptions(new, old))

    # Can't update objects
    new = '{"widget":"TextBox","cant":{"prop":1}}'
    old = '{"widget":"TextBox","cant":{"prop":2}}'
    self.assertFalse(widgetOptions(new, old))

    # Can't update lists
    new = '{"widget":"TextBox","cant":[1, 2]}'
    old = '{"widget":"TextBox","cant":[2, 1]}'
    self.assertFalse(widgetOptions(new, old))

    # Can update lists
    new = '{"widget":"TextBox","alignment":[1, 2]}'
    old = '{"widget":"TextBox","alignment":[3, 2]}'
    self.assertTrue(widgetOptions(new, old))

    # Can update without changing list.
    new = '{"widget":"TextBox","dontChange":[1, 2]}'
    old = '{"widget":"Spinner","dontChange":[1, 2]}'
    self.assertTrue(widgetOptions(new, old))
  # pylint: enable=R0915