(core) Fix error in sandbox when removing multiple summary source columns

Summary:
Fixes a very specific bug reported here: https://grist.slack.com/archives/C069RUP71/p1694630242765769

The error occurred when:

1. Removing multiple columns simultaneously
2. Those columns were sources of groupby columns for a summary table (so removing them meant recreating the summary table and thus deleting its columns)
3. There was a display column for one of the columns that got deleted (either directly or indirectly) which was set to be automatically removed since it was no longer needed, but this failed because the column was already deleted as part of earlier table removal.

I fixed this by making `apply_auto_removes` remove table records last, so removing the display column wouldn't be a problem.

That fixed the original error, but then I noticed that trying to undo the removal could lead to another error (under even more specific circumstances). It's hard to see exactly why, but I can see that just 3 `RemoveColumn` user actions generated over 100 doc actions and corresponding undo actions, hence the difficulty in narrowing the problem down. This is partly because removing a single column would recreate a summary table, only for that table to be immediately replaced again when another column was removed. Making the frontend send a single `[BulkRemoveRecord, _grist_Tables_column, ...] ` leads to a more efficient and sensible process with about half as many doc actions and no undo error. I think this alone would also solve the original error, but the data engine change seems more generally helpful and worth keeping.

Test Plan: Added a Python test and an nbrowser test

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D4052

app/client/components/GridView.js

@@ -890,10 +890,14 @@ GridView.prototype.deleteColumns = function(selection) {
     });
     return Promise.resolve(false);
   }
-  let actions = fields.filter(col => !col.disableModify()).map(col => ['RemoveColumn', col.colId()]);
-  if (actions.length > 0) {
-    return this.tableModel.sendTableActions(actions, `Removed columns ${actions.map(a => a[1]).join(', ')} ` +
-      `from ${this.tableModel.tableData.tableId}.`).then(() => this.clearSelection());
+  const columns = fields.filter(col => !col.disableModify());
+  const colRefs = columns.map(col => col.colRef.peek());
+  if (colRefs.length > 0) {
+    return this.gristDoc.docData.sendAction(
+        ['BulkRemoveRecord', '_grist_Tables_column', colRefs],
+        `Removed columns ${columns.map(col => col.colId.peek()).join(', ')} ` +
+        `from ${this.tableModel.tableData.tableId}.`
+    ).then(() => this.clearSelection());
   }
   return Promise.resolve(false);
 };

sandbox/grist/docmodel.py

@@ -260,7 +260,11 @@ class DocModel(object):
     Remove the records marked for removal.
     """
     # Sort to make sure removals are done in deterministic order.
-    gone_records = sorted(self._auto_remove_set)
+    gone_records = sorted(
+      self._auto_remove_set,
+      # Remove tables last to prevent errors trying to remove rows or columns from deleted tables.
+      key=lambda r: (r._table.table_id == "_grist_Tables", r)
+    )
     self._auto_remove_set.clear()
     # setAutoRemove is called by formulas, notably summary tables, and shouldn't be blocked by ACL.
     with self._engine.user_actions.indirect_actions():

sandbox/grist/test_summary.py

@@ -4,10 +4,12 @@ files: test_summary.py and test_summary2.py.
 """
 
 import logging
+
 import actions
 import summary
-import testutil
 import test_engine
+import testutil
+import useractions
 from test_engine import Table, Column, View, Section, Field
 from useractions import allowed_summary_change
 
@@ -870,6 +872,57 @@ class Address:
       [ 2,    2.0,  [3],      1,        6   ],
     ])
 
+  def test_remove_source_columns_with_display_column(self):
+    # Verify a fix for a specific bug: removing multiple groupby source columns
+    # when the summary table contains a display column.
+
+    self.apply_user_action(["AddEmptyTable", None])
+    # Group by A and B
+    self.apply_user_action(["CreateViewSection", 1, 0, "record", [2,3], None])
+    # Add a display column for the group column
+    self.apply_user_action(['SetDisplayFormula', 'Table1_summary_A_B', None, 7, '$group.C'])
+
+    # Verify metadata and initially.
+    self.assertTables([
+      Table(1, "Table1", summarySourceTable=0, primaryViewId=1, columns=[
+        Column(1, "manualSort", "ManualSortPos",  False,  "", 0),
+        Column(2, "A",          "Any",        True,  "", 0),
+        Column(3, "B",          "Any",        True,  "", 0),
+        Column(4, "C",          "Any",        True,  "", 0),
+      ]),
+      Table(2, "Table1_summary_A_B", summarySourceTable=1, primaryViewId=0, columns=[
+        Column(5, "A",          "Any",            False, "", 2),
+        Column(6, "B",          "Any",            False, "", 3),
+        Column(7, "group",      "RefList:Table1", True,  "table.getSummarySourceGroup(rec)", 0),
+        Column(8, "count",      "Int",            True,  "len($group)", 0),
+        Column(9, "gristHelper_Display", "Any",   True,  "$group.C", 0),
+      ])
+    ])
+
+    user_actions = [
+      useractions.from_repr(ua) for ua in
+      [
+        ['RemoveColumn', 'Table1', 'A'],
+        ['RemoveColumn', 'Table1', 'B'],
+      ]
+    ]
+    self.engine.apply_user_actions(user_actions)
+
+    # Verify that the final structure is as expected.
+    self.assertTables([
+      Table(1, "Table1", summarySourceTable=0, primaryViewId=1, columns=[
+        Column(1, "manualSort", "ManualSortPos",  False,  "", 0),
+        Column(4, "C",          "Any",        True,  "", 0),
+      ]),
+      # Table1_summary_A_B was removed and recreated as Table1_summary.
+      # This removed Table1_summary_A_B.group which automatically removed gristHelper_Display
+      # which led to an error in the past.
+      Table(4, "Table1_summary", summarySourceTable=1, primaryViewId=0, columns=[
+        Column(14, "group",      "RefList:Table1", True,  "table.getSummarySourceGroup(rec)", 0),
+        Column(15, "count",      "Int",            True,  "len($group)", 0),
+      ])
+    ])
+
   #----------------------------------------------------------------------
   # pylint: disable=R0915
   def test_allow_select_by_change(self):

test/nbrowser/DeleteColumnsUndo.ts

@@ -0,0 +1,33 @@
+import {assert, driver, Key} from 'mocha-webdriver';
+
+import * as gu from 'test/nbrowser/gristUtils';
+import {server, setupTestSuite} from "test/nbrowser/testUtils";
+
+describe('DeleteColumnsUndo', function () {
+  this.timeout(20000);
+  setupTestSuite();
+
+  before(async function () {
+    await server.simulateLogin("Chimpy", "chimpy@getgrist.com", 'nasa');
+    const doc = await gu.importFixturesDoc('chimpy', 'nasa', 'Horizon', 'DeleteColumnsUndo.grist', false);
+    await driver.get(`${server.getHost()}/o/nasa/doc/${doc.id}/p/2`);
+    await gu.waitForDocToLoad();
+  });
+
+  it('should be able to delete multiple columns and undo without errors', async function () {
+    const revert = await gu.begin();
+    assert.deepEqual(await gu.getColumnNames(), ['A', 'B', 'C', 'D']);
+    await gu.getColumnHeader({col: 'A'}).click();
+    await gu.sendKeys(Key.chord(Key.SHIFT, Key.RIGHT));
+    await gu.sendKeys(Key.chord(Key.SHIFT, Key.RIGHT));
+    const selectedCols = await driver.findAll(".column_name.selected");
+    assert.lengthOf(selectedCols, 3);
+    await gu.openColumnMenu('A', 'Delete 3 columns');
+    await gu.waitForServer();
+    assert.deepEqual(await gu.getColumnNames(), ['D']);
+    await revert();
+    await gu.checkForErrors();
+    assert.deepEqual(await gu.getColumnNames(), ['A', 'B', 'C', 'D']);
+  });
+
+});