(core) Allow using negative rowIds to add records and refer to them in Reference values.

Summary:
- When adding records, negative rowIds may now be specified. They'll be replaced by proper IDs.
- If these negative IDs are used in Reference columns in subsequent actions in
  the same bundle of UserActions, they get replaced with the proper rowIds.
- Use this to sync ACLResources and ACLRules from UI in a single batch of UserActions.
- Resolve the TODOs in GranularAccess test, to no longer need to guess resource rowIds.

Test Plan: Added a python unittest for mapping IDs; updated browser tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2691

sandbox/grist/action_summary.py

@@ -52,6 +52,23 @@ class ActionSummary(object):
     col_delta = table_delta and table_delta.column_deltas.pop(col_id, None)
     return self._changes_to_actions(table_id, col_id, col_delta or {}, out_stored, out_undo)
 
+  def update_new_rows_map(self, table_id, temp_row_ids, final_row_ids):
+    """
+    Add a mapping from temporary negative row_ids to the final ones, for rows added to the given
+    table. The two lists must have the same length; only negative row_ids are remembered. If a
+    negative row_id was already used, its mapping will be overridden.
+    """
+    t = self._forTable(table_id)
+    t.temp_row_ids.update((a, b) for (a, b) in zip(temp_row_ids, final_row_ids) if a and a < 0)
+
+  def translate_new_row_ids(self, table_id, row_ids):
+    """
+    Translate any temporary (negative) row_ids to their final values, using mappings created by
+    update_new_rows_map().
+    """
+    t = self._forTable(table_id)
+    return [t.temp_row_ids.get(r, r) for r in row_ids]
+
   def _changes_to_actions(self, table_id, col_id, column_delta, out_stored, out_undo):
     """
     Given a column and a dict of column_deltas for it, of the form {row_id: (before_value,
@@ -155,6 +172,10 @@ class TableDelta(object):
     self.column_renames = LabelRenames()
     self.column_deltas = {}   # maps col_id to the dict {row_id: (before_value, after_value)}
 
+    # Map of negative row_ids that may be used in [Bulk]AddRecord actions to the final row_ids for
+    # those rows; to allow translating Reference values added in the same action bundle.
+    self.temp_row_ids = {}
+
 
 class LabelRenames(object):
   """

sandbox/grist/column.py

@@ -201,7 +201,7 @@ class BaseColumn(object):
     """
     return self.type_obj.convert(value_to_convert)
 
-  def prepare_new_values(self, values, ignore_data=False):
+  def prepare_new_values(self, values, ignore_data=False, action_summary=None):
     """
     This allows us to modify values and also produce adjustments to existing records. This
     currently is only used by PositionColumn. Returns two lists: new_values, and
@@ -281,7 +281,7 @@ class PositionColumn(NumericColumn):
     super(PositionColumn, self).copy_from_column(other_column)
     self._sorted_rows = SortedListWithKey(other_column._sorted_rows[:], key=self.raw_get)
 
-  def prepare_new_values(self, values, ignore_data=False):
+  def prepare_new_values(self, values, ignore_data=False, action_summary=None):
     # This does the work of adjusting positions and relabeling existing rows with new position
     # (without changing sort order) to make space for the new positions. Note that this is also
     # used for updating a position for an existing row: we'll find a new value for it; later when
@@ -356,6 +356,11 @@ class ReferenceColumn(BaseReferenceColumn):
     if new_value:
       self._relation.add_reference(row_id, new_value)
 
+  def prepare_new_values(self, values, ignore_data=False, action_summary=None):
+    if action_summary and values:
+      values = action_summary.translate_new_row_ids(self._target_table.table_id, values)
+    return values, []
+
 
 class ReferenceListColumn(BaseReferenceColumn):
   """

sandbox/grist/engine.py

@@ -862,7 +862,8 @@ class Engine(object):
 
       # If there are values for any PositionNumber columns, ensure PositionNumbers are ordered as
       # intended but are all unique, which may require updating other positions.
-      nvalues, adjustments = col_obj.prepare_new_values(values)
+      nvalues, adjustments = col_obj.prepare_new_values(values,
+          action_summary=self.out_actions.summary)
       if adjustments:
         extra_actions.append(actions.BulkUpdateRecord(
           action.table_id, [r for r,v in adjustments], {col_id: [v for r,v in adjustments]}))
@@ -880,7 +881,8 @@ class Engine(object):
         defaults = [col_obj.getdefault() for r in row_ids]
         # We use defaults to get new values or adjustments. If we are replacing data, we'll make
         # the adjustments without regard to the existing data.
-        nvalues, adjustments = col_obj.prepare_new_values(defaults, ignore_data=ignore_data)
+        nvalues, adjustments = col_obj.prepare_new_values(defaults, ignore_data=ignore_data,
+            action_summary=self.out_actions.summary)
         if adjustments:
           extra_actions.append(actions.BulkUpdateRecord(
             action.table_id, [r for r,v in adjustments], {col_id: [v for r,v in adjustments]}))

sandbox/grist/test_temp_rowids.py

@@ -0,0 +1,88 @@
+import test_engine
+import testsamples
+import useractions
+
+class TestTempRowIds(test_engine.EngineTestCase):
+
+  def test_temp_row_ids(self):
+    self.load_sample(testsamples.sample_students)
+
+    out_actions = self.engine.apply_user_actions([useractions.from_repr(ua) for ua in (
+      # Add a mix of records with or without temp rowIds.
+      ['AddRecord', 'Address', None, {'city': 'A'}],
+      ['AddRecord', 'Address', -1, {'city': 'B'}],
+      ['BulkAddRecord', 'Address', [-3, None, -7, -10], {'city': ['C', 'D', 'E', 'F']}],
+
+      # -3 translates to C; the new record of -1 applies to a different table, so doesn't affect
+      # its translation to city A.
+      ['AddRecord', 'Schools', -1, {'address': -3, 'name': 'SUNY C'}],
+
+      # Add a mix of records referring to new, existing, or null rows.
+      ['BulkAddRecord', 'Schools', [None, None, None, None, None], {
+        'address': [-1, 11, 0, -3, -7],
+        'name': ['SUNY A', 'NYU', 'Xavier', 'Suny C2', 'Suny E'],
+        }
+      ],
+
+      # Try a few updates too.
+      ['UpdateRecord', 'Schools', 1, {'address': -7}],
+      ['BulkUpdateRecord', 'Schools', [2, 3, 4], {'address': [-3, -1, 11]}],
+
+      # Later temp rowIds override previous one. Here, -3 was already used.
+      ['AddRecord', 'Address', -3, {'city': 'G'}],
+      ['AddRecord', 'Schools', None, {'address': -3, 'name': 'SUNY G'}],
+    )])
+
+    # Test that we get the correct resulting data.
+    self.assertTableData('Address', cols="subset", data=[
+      ["id",  "city"       ],
+      [11,    "New York"   ],
+      [12,    "Colombia"   ],
+      [13,    "New Haven"  ],
+      [14,    "West Haven" ],
+      [15,    "A"],
+      [16,    "B"],   # was -1
+      [17,    "C"],   # was -3
+      [18,    "D"],
+      [19,    "E"],   # was -7
+      [20,    "F"],   # was -10
+      [21,    "G"],   # was -3
+    ])
+    self.assertTableData('Schools', cols="subset", data=[
+      ["id",  "name",     "address"],
+      [1, "Columbia",     19],
+      [2, "Columbia",     17],
+      [3, "Yale",         16],
+      [4, "Yale",         11],
+      [5, "SUNY C",       17],
+      [6, "SUNY A",       16],
+      [7, "NYU",          11],
+      [8, "Xavier",       0],
+      [9, "Suny C2",      17],
+      [10, "Suny E",      19],
+      [11, "SUNY G",      21],
+    ])
+
+    # Test that the actions above got properly translated.
+    # These are same as above, except for the translated rowIds.
+    self.assertPartialOutActions(out_actions, {
+      "stored": [
+        ['AddRecord', 'Address', 15, {'city': 'A'}],
+        ['AddRecord', 'Address', 16, {'city': 'B'}],
+        ['BulkAddRecord', 'Address', [17, 18, 19, 20], {'city': ['C', 'D', 'E', 'F']}],
+        ['AddRecord', 'Schools', 5, {'address': 17, 'name': 'SUNY C'}],
+        ['BulkAddRecord', 'Schools', [6, 7, 8, 9, 10], {
+          'address': [16, 11, 0, 17, 19],
+          'name': ['SUNY A', 'NYU', 'Xavier', 'Suny C2', 'Suny E'],
+          }
+        ],
+        ['UpdateRecord', 'Schools', 1, {'address': 19}],
+        ['BulkUpdateRecord', 'Schools', [2, 3, 4], {'address': [17, 16, 11]}],
+        ['AddRecord', 'Address', 21, {'city': 'G'}],
+        ['AddRecord', 'Schools', 11, {'address': 21, 'name': 'SUNY G'}],
+
+        # Calculated values (for Students; lookups on schools named "Columbia" and "Yale")
+        ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 6], {
+          "schoolCities": ["E:C", "B:New York", "E:C", "B:New York", "B:New York"]}],
+      ]
+    })

sandbox/grist/useractions.py

@@ -294,10 +294,15 @@ class UserActions(object):
     # Make a copy of row_ids and fill in those set to None.
     filled_row_ids = row_ids[:]
     for i, row_id in enumerate(filled_row_ids):
-      if row_id is None:
+      if row_id is None or row_id < 0:
         filled_row_ids[i] = row_id = next_row_id
       next_row_id = max(next_row_id, row_id) + 1
 
+    # Whenever we add new rows, remember the mapping from any negative row_ids to their final
+    # values. This allows the negative_row_ids to be used as Reference values in subsequent
+    # actions in the same bundle.
+    self._engine.out_actions.summary.update_new_rows_map(table_id, row_ids, filled_row_ids)
+
     # Convert entered values to the correct types.
     ActionType = actions.ReplaceTableData if replace else actions.BulkAddRecord
     action, extra_actions = self._engine.convert_action_values(