(core) Implement trigger formulas (generalizing default formulas)

Summary:
Trigger formulas can be calculated for new records, or for new records and
updates to certain fields, or all fields. They do not recalculate on open,
and they MAY be set directly by the user, including for data-cleaning.

- Column metadata now includes recalcWhen and recalcDeps fields.
- Trigger formulas are NOT recalculated on open or on schema changes.
- When recalcWhen is "never", formula isn't calculated even for new records.
- When recalcWhen is "allupdates", formula is calculated for new records and
  any manual (non-formula) updates to the record.
- When recalcWhen is "", formula is calculated for new records, and changes to
  recalcDeps fields (which may be formula fields or column itself).
- A column whose recalcDeps includes itself is a "data-cleaning" column; a
  value set by the user will still trigger the formula.
- All trigger-formulas receive a "value" argument (to support the case above).

Small changes
- Update RefLists (used for recalcDeps) when target rows are deleted.
- Add RecordList.__contains__ (for `rec in refList` or `id in refList` checks)
- Clarify that Calculate action has replaced load_done() in practice,
  and use it in tests too, to better match reality.

Left for later:
- UI for setting recalcWhen / recalcDeps.
- Implementation of actions such as "Recalculate for all cells".
- Allowing trigger-formulas access to the current user's info.

Test Plan: Added a comprehensive python-side test for various trigger combinations

Reviewers: paulfitz, alexmojaki

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2872

app/common/schema.ts

@@ -31,6 +31,8 @@ export const schema = {
     summarySourceCol    : "Ref:_grist_Tables_column",
     displayCol          : "Ref:_grist_Tables_column",
     visibleCol          : "Ref:_grist_Tables_column",
+    recalcWhen          : "Text",
+    recalcDeps          : "RefList:_grist_Tables_column",
   },
 
   "_grist_Imports": {
@@ -202,6 +204,8 @@ export interface SchemaTypes {
     summarySourceCol: number;
     displayCol: number;
     visibleCol: number;
+    recalcWhen: string;
+    recalcDeps: number[];
   };
 
   "_grist_Imports": {

sandbox/gen_js_schema.py

@@ -16,6 +16,7 @@ _ts_types = {
   "Int":            "number",
   "PositionNumber": "number",
   "Ref":            "number",
+  "RefList":        "number[]",
   "Text":           "string",
 }
 

sandbox/grist/column.py

@@ -71,7 +71,7 @@ class BaseColumn(object):
     self.node = depend.Node(self.table_id, col_id)
     self._is_formula = col_info.is_formula
     self._is_private = bool(col_info.method) and getattr(col_info.method, 'is_private', False)
-    self.method = col_info.method
+    self.update_method(col_info.method)
 
     # Always initialize to include the special empty record at index 0.
     self.growto(1)
@@ -82,7 +82,11 @@ class BaseColumn(object):
     'method' function. The method may refer to variables in the generated "usercode" module, and
     it's important that all such references are to the rebuilt "usercode" module.
     """
-    self.method = method
+    if not self._is_formula and method:
+      # Include the current value of the cell as the third parameter (to default formulas).
+      self.method = lambda rec, table: method(rec, table, self.get_cell_value(int(rec)))
+    else:
+      self.method = method
 
   def is_formula(self):
     """
@@ -394,6 +398,21 @@ class BaseReferenceColumn(BaseColumn):
   def sample_value(self):
     return self._target_table.sample_record
 
+  def get_updates_for_removed_target_rows(self, target_row_ids):
+    """
+    Returns a list of pairs of (row_id, new_value) for values in this column that need to be
+    updated when target_row_ids are removed from the referenced table.
+    """
+    affected_rows = sorted(self._relation.get_affected_rows(target_row_ids))
+    return [(row_id, self._raw_get_without(row_id, target_row_ids)) for row_id in affected_rows]
+
+  def _raw_get_without(self, _row_id, _target_row_ids):
+    """
+    Returns a Ref or RefList cell value with the specified target_row_ids removed, assuming one of
+    them is actually present in the value. For References, it just leaves the default value.
+    """
+    return self.getdefault()
+
 
 class ReferenceColumn(BaseReferenceColumn):
   """
@@ -439,6 +458,14 @@ class ReferenceListColumn(BaseReferenceColumn):
       return typed_value
     return self._target_table.RecordSet(self._target_table, typed_value, self._relation)
 
+  def _raw_get_without(self, row_id, target_row_ids):
+    """
+    Returns the RefList cell value at row_id with the specified target_row_ids removed.
+    """
+    raw = self.raw_get(row_id)
+    if self.type_obj.is_right_type(raw):
+      raw = [r for r in raw if r not in target_row_ids] or None
+    return raw
 
 # Set up the relationship between usertypes objects and column objects.
 usertypes.BaseColumnType.ColType = DataColumn

sandbox/grist/depend.py

@@ -155,8 +155,7 @@ class Graph(object):
       include_self = True
 
       for edge in self._in_node_map.get(dirty_node, ()):
-        affected_rows = (ALL_ROWS if dirty_rows == ALL_ROWS else
+        affected_rows = edge.relation.get_affected_rows(dirty_rows)
-                         edge.relation.get_affected_rows(dirty_rows))
 
         # Previously this was:
         #   self.invalidate_deps(edge.out_node, affected_rows, recompute_map, include_self=True)

sandbox/grist/docactions.py

@@ -80,6 +80,12 @@ class DocActions(object):
       for (row_id, value) in zip(row_ids, values):
         col.set(row_id, value)
 
+      # Non-formula columns may get invalidated and recalculated if they have a trigger formula.
+      # Prevent such recalculation if we set an explicit value for them (we want to prevent it
+      # even if triggered by something else within the same useraction).
+      if not col.is_formula():
+        self._engine.prevent_recalc(col.node, row_ids, should_prevent=True)
+
     # Generate the undo action.
     self._engine.out_actions.undo.append(
         actions.BulkUpdateRecord(table_id, row_ids, undo_values).simplify())

sandbox/grist/docmodel.py

@@ -14,6 +14,7 @@ import usertypes
 import relabeling
 import table
 import moment
+from schema import RecalcWhen
 
 # pylint:disable=redefined-outer-name
 
@@ -83,6 +84,13 @@ class MetaTableExtras(object):
       """
       return len(rec.usedByCols) + len(rec.usedByFields)
 
+    def recalcOnChangesToSelf(rec, table):
+      """
+      Whether the column is a trigger-formula column that depends on itself, used for
+      data-cleaning. (A manual change to it will trigger its own recalculation.)
+      """
+      return rec.recalcWhen == RecalcWhen.DEFAULT and rec.id in rec.recalcDeps
+
     def setAutoRemove(rec, table):
       """Marks the col for removal if it's a display helper col with no more users."""
       table.docmodel.setAutoRemove(rec,

sandbox/grist/engine.py

@@ -29,7 +29,9 @@ import logger
 import match_counter
 import objtypes
 from objtypes import strict_equal
+from relation import SingleRowsIdentityRelation
 import schema
+from schema import RecalcWhen
 import table as table_module
 import useractions
 import column
@@ -119,6 +121,7 @@ class Engine(object):
         - Then load_table() must be called once for each of the other tables (both special tables,
           and user tables), with that table's data (no need to call it for empty tables).
         - Finally, load_done() must be called once to finish initialization.
+          NOTE: instead of load_done(), Grist now applies the no-op 'Calculate' user action.
 
     Other methods:
 
@@ -218,6 +221,13 @@ class Engine(object):
     # Create the object that knows how to interpret UserActions.
     self.user_actions = useractions.UserActions(self)
 
+    # Map from node to set of row_ids, for cells that should not be recalculated because they are
+    # data columns manually changed in this UserAction.
+    self._prevent_recompute_map = {}
+
+    # Whether any trigger columns may need to have their dependencies rebuilt.
+    self._have_trigger_columns_changed = True
+
     # A flag for when a useraction causes a schema change, to verify consistency afterwards.
     self._schema_updated = False
 
@@ -278,6 +288,7 @@ class Engine(object):
   def load_done(self):
     """
     Finalizes the loading of data into this Engine.
+    NOTE: instead of load_done(), Grist now applies the no-op 'Calculate' user action.
     """
     self._bring_all_up_to_date()
 
@@ -728,6 +739,11 @@ class Engine(object):
     if dirty_rows == depend.ALL_ROWS:
       dirty_rows = SortedSet(r for r in table.row_ids if r not in exclude)
       self.recompute_map[node] = dirty_rows
+
+    exempt = self._prevent_recompute_map.get(node, None)
+    if exempt:
+      dirty_rows.difference_update(exempt)
+
     require_rows = sorted(require_rows or [])
 
     # Prevents dependency creation for non-formula nodes. A non-formula column may include a
@@ -984,6 +1000,13 @@ class Engine(object):
     self.dep_graph.invalidate_deps(col_obj.node, row_ids, self.recompute_map,
                                    include_self=include_self)
 
+  def prevent_recalc(self, node, row_ids, should_prevent):
+    prevented = self._prevent_recompute_map.setdefault(node, set())
+    if should_prevent:
+      prevented.update(row_ids)
+    else:
+      prevented.difference_update(row_ids)
+
   def rebuild_usercode(self):
     """
     Compiles the usercode from the schema, and updates all tables and columns to match.
@@ -1015,6 +1038,9 @@ class Engine(object):
     # Update docmodel with references to the updated metadata tables.
     self.docmodel.update_tables()
 
+    # Set flag to rebuild dependencies of trigger columns after any potential renames, etc.
+    self.trigger_columns_changed()
+
     # The order here is important to make sure that when we update the usercode,
     # we don't overwrite with outdated usercode entries
     self._repl.locals.update(self.gencode.usercode.__dict__)
@@ -1023,6 +1049,8 @@ class Engine(object):
     # Update the context used for autocompletions.
     self._autocomplete_context = AutocompleteContext(self.gencode.usercode.__dict__)
 
+  def trigger_columns_changed(self):
+    self._have_trigger_columns_changed = True
 
   def _update_table_model(self, table, user_table):
     """
@@ -1058,6 +1086,35 @@ class Engine(object):
       for c in table.get_helper_columns():
         self.delete_column(c)
 
+  def _maybe_update_trigger_dependencies(self):
+    if not self._have_trigger_columns_changed:
+      return
+    self._have_trigger_columns_changed = False
+
+    # Without being very smart, if trigger-formula dependencies change for any columns, rebuild
+    # them for all columns. Specifically, we will create nodes and edges in the dependency graph.
+    for table_id, table in self.tables.iteritems():
+      if table_id.startswith('_grist_'):
+        # We can skip metadata tables, there are no trigger-formulas there.
+        continue
+      for col_id, col_obj in table.all_columns.iteritems():
+        if col_obj.is_formula() or not col_obj.has_formula():
+          continue
+        col_rec = self.docmodel.columns.lookupOne(tableId=table_id, colId=col_id)
+
+        out_node = depend.Node(table_id, col_id)
+        rel = SingleRowsIdentityRelation(table_id)
+        self.dep_graph.clear_dependencies(out_node)
+
+        # When we have explicit dependencies, add them to dep_graph.
+        if col_rec.recalcWhen == RecalcWhen.DEFAULT:
+          for dc in col_rec.recalcDeps:
+            in_node = depend.Node(table_id, dc.colId)
+            edge = depend.Edge(out_node, in_node, rel)
+            if edge not in self._recompute_edge_set:
+              self._recompute_edge_set.add(edge)
+              self.dep_graph.add_edge(*edge)
+
 
   def delete_column(self, col_obj):
     # Remove the column from its table.
@@ -1067,7 +1124,8 @@ class Engine(object):
     # Invalidate anything that depends on the column being deleted. The column may be gone from
     # the table itself, so we use invalidate_column directly.
     self.invalidate_column(col_obj)
-    # Remove reference to the column from the recompute_map.
+    # Remove reference to the column from the dependency graph and the recompute_map.
+    self.dep_graph.clear_dependencies(col_obj.node)
     self.recompute_map.pop(col_obj.node, None)
     # Mark the column to be destroyed at the end of applying this docaction.
     self._gone_columns.append(col_obj)
@@ -1103,6 +1161,11 @@ class Engine(object):
     try:
       for user_action in user_actions:
         self._schema_updated = False
+
+        # At the start of each useraction, clear exemptions. These are used to avoid recalcs of
+        # trigger-formula columns for which the same useractions sets an explicit value.
+        self._prevent_recompute_map.clear()
+
         self.out_actions.retValues.append(self._apply_one_user_action(user_action))
 
         # If the UserAction touched the schema, check that it is now consistent with metadata.
@@ -1135,6 +1198,9 @@ class Engine(object):
       else:
         raise
 
+    # If needed, rebuild dependencies for trigger formulas.
+    self._maybe_update_trigger_dependencies()
+
     # Note that recalculations and auto-removals get included after processing all useractions.
     self._bring_all_up_to_date()
 

sandbox/grist/gencode.py

@@ -75,12 +75,13 @@ class GenCode(object):
     self._user_builder = None
     self._usercode = None
 
-  def _make_formula_field(self, col_info, table_id, name=None, include_type=True):
+  def _make_formula_field(self, col_info, table_id, name=None, include_type=True,
+      include_value_arg=False):
     """Returns the code for a formula field."""
     # If the caller didn't specify a special name, use the colId
     name = name or col_info.colId
 
-    decl = "def %s(rec, table):\n" % name
+    decl = "def %s(rec, table%s):\n" % (name, (", value" if include_value_arg else ""))
 
     # This is where we get to use the formula cache, and save the work of rebuilding formulas.
     key = (table_id, col_info.colId, col_info.formula)
@@ -102,7 +103,8 @@ class GenCode(object):
     if col_info.formula:
       parts.append(self._make_formula_field(col_info, table_id,
                                             name=table.get_default_func_name(col_info.colId),
-                                            include_type=False))
+                                            include_type=False,
+                                            include_value_arg=True))
     parts.append("%s = %s\n" % (col_info.colId, get_grist_type(col_info.type)))
     return textbuilder.Combiner(parts)
 

sandbox/grist/lookup.py

@@ -172,6 +172,8 @@ class _LookupRelation(relation.Relation):
     return "_LookupRelation(%s->%s)" % (self._referring_node, self.target_table)
 
   def get_affected_rows(self, target_row_ids):
+    if target_row_ids == depend.ALL_ROWS:
+      return depend.ALL_ROWS
     # Each target row (result of a lookup by key) is associated with a key, and all rows that
     # looked up an affected key are affected by a change to any associated row. We remember which
     # rows looked up which key in self._row_key_map, so that when some target row changes to a new

sandbox/grist/migrations.py

@@ -780,3 +780,11 @@ def migration21(tdset):
     add_column('_grist_ACLRules', 'rulePos', 'PositionNumber'),
     add_column('_grist_ACLRules', 'userAttributes', 'Text'),
   ])
+
+
+@migration(schema_version=22)
+def migration22(tdset):
+  return tdset.apply_doc_actions([
+    add_column('_grist_Tables_column', 'recalcWhen', 'Int'),
+    add_column('_grist_Tables_column', 'recalcDeps', 'RefList:_grist_Tables_column'),
+  ])

sandbox/grist/records.py

@@ -150,8 +150,7 @@ class RecordSet(object):
 
   def __eq__(self, other):
     return (isinstance(other, RecordSet) and
-        (self._table, self._row_ids, self._group_by, self._sort_by) ==
+        (self._table, self._row_ids) == (other._table, other._row_ids))
-        (other._table, other._row_ids, other._group_by, other._sort_by))
 
   def __ne__(self, other):
     return not self.__eq__(other)
@@ -160,6 +159,14 @@ class RecordSet(object):
     for row_id in self._row_ids:
       yield self.Record(self._table, row_id, self._source_relation)
 
+  def __contains__(self, item):
+    """item may be a Record or its row_id."""
+    if isinstance(item, int):
+      return item in self._row_ids
+    if isinstance(item, Record) and item._table == self._table:
+      return int(item) in self._row_ids
+    return False
+
   def get_one(self):
     row_id = min(self._row_ids) if self._row_ids else 0
     return self.Record(self._table, row_id, self._source_relation)

sandbox/grist/relation.py

@@ -65,6 +65,19 @@ class IdentityRelation(Relation):
   # test_dependencies_relations_bug for a detailed description of a bug this can cause.]
 
 
+class SingleRowsIdentityRelation(IdentityRelation):
+  """
+  Represents an identity relation, but one which refuses to pass along ALL_ROWS. In other words,
+  if a full column changed (i.e. ALL_ROWS), none of the dependent cells will be considered
+  changed. But when specific rows are changed, those changes propagate.
+
+  This is used for trigger formulas, to ensure they don't recalculate in full when a dependency
+  column is renamed or modified (as opposed to particular records).
+  """
+  def get_affected_rows(self, input_rows):
+    return [] if input_rows == depend.ALL_ROWS else input_rows
+
+
 class ComposedRelation(Relation):
   """
   Represents a composition of two Relations. E.g. if referring side maps Students to Schools, and
@@ -107,6 +120,8 @@ class ReferenceRelation(Relation):
   def get_affected_rows(self, input_rows):
     # Each input row (target of the reference link) may be pointed to by multiple references,
     # so we need to take the union of all of those sets.
+    if input_rows == depend.ALL_ROWS:
+      return depend.ALL_ROWS
     affected_rows = set()
     for target_row_id in input_rows:
       affected_rows.update(self.inverse_map.get(target_row_id, ()))

sandbox/grist/schema.py

@@ -15,7 +15,7 @@ import six
 
 import actions
 
-SCHEMA_VERSION = 21
+SCHEMA_VERSION = 22
 
 def make_column(col_id, col_type, formula='', isFormula=False):
   return {
@@ -78,6 +78,14 @@ def schema_create_actions():
       # E.g. Foo.person may have a visibleCol pointing to People.Name, with the displayCol
       # pointing to Foo._gristHelper_DisplayX column with the formula "$person.Name".
       make_column("visibleCol",       "Ref:_grist_Tables_column"),
+
+      # Instructions when to recalculate the formula on a column with isFormula=False (previously
+      # known as a "default formula"). Values are RecalcWhen constants defined below.
+      make_column("recalcWhen",       "Int"),
+
+      # List of fields that should trigger a calculation of a formula in a data column. Only
+      # applies when recalcWhen is RecalcWhen.DEFAULT, and defaults to the empty list.
+      make_column("recalcDeps",       "RefList:_grist_Tables_column"),
     ]),
 
     # DEPRECATED: Previously used to keep import options, and allow the user to change them.
@@ -285,6 +293,17 @@ def schema_create_actions():
   ]
 
 
+class RecalcWhen(object):
+  """
+  Constants for column's recalcWhen field, which determine when a formula associated with a data
+  column would get calculated.
+  """
+  DEFAULT = 0         # Calculate on new records or when any field in recalcDeps changes. If
+                      # recalcDeps includes this column itself, it's a "data-cleaning" formula.
+  NEVER = 1           # Don't calculate automatically (but user can trigger manually)
+  MANUAL_UPDATES = 2  # Calculate on new records and on manual updates to any data field.
+
+
 # These are little structs to represent the document schema that's used in code generation.
 # Schema itself (as stored by Engine) is an OrderedDict(tableId -> SchemaTable), with
 # SchemaTable.columns being an OrderedDict(colId -> SchemaColumn).

sandbox/grist/test_engine.py

@@ -292,7 +292,9 @@ class EngineTestCase(unittest.TestCase):
     self.engine.load_meta_tables(schema['_grist_Tables'], schema['_grist_Tables_column'])
     for data in six.itervalues(sample["DATA"]):
       self.engine.load_table(data)
-    self.engine.load_done()
+    # We used to call load_done() at the end; in practice, Grist's ActiveDoc does not call
+    # load_done, but applies the "Calculate" user action. Do that for more realistic tests.
+    self.apply_user_action(['Calculate'])
 
   # The following are convenience methods for tests deriving from EngineTestCase.
   def add_column(self, table_name, col_name, **kwargs):

sandbox/grist/test_recordlist.py

@@ -0,0 +1,85 @@
+import testutil
+import test_engine
+
+class TestRecordList(test_engine.EngineTestCase):
+  col = testutil.col_schema_row
+  sample_desc = {
+    "SCHEMA": [
+      [1, "Creatures", [
+        col(1, "Name",   "Text", False),
+        col(2, "Class",  "Ref:Class", False),
+      ]],
+      [2, "Class", [
+        col(11, "Name",       "Text",               False),
+        col(12, "Creatures",  "RefList:Creatures",  False),
+      ]],
+    ],
+    "DATA": {
+      "Class": [
+        ["id", "Name", "Creatures"],
+        [1, "Mammals",  [1, 3]],
+        [2, "Reptilia", [2, 4]],
+      ],
+      "Creatures": [
+        ["id","Name",    "Class"],
+        [1,   "Cat",     1],
+        [2,   "Chicken", 2],
+        [3,   "Dolphin", 1],
+        [4,   "Turtle",  2],
+      ],
+    }
+  }
+  sample = testutil.parse_test_sample(sample_desc)
+
+  def test_removals(self):
+    # Removing target rows should remove them from RefList columns.
+    self.load_sample(self.sample)
+    self.assertTableData("Class", data=[
+      ["id", "Name", "Creatures"],
+      [1, "Mammals",  [1, 3]],
+      [2, "Reptilia", [2, 4]],
+    ])
+
+    self.remove_record("Creatures", 2)
+    self.assertTableData("Class", data=[
+      ["id", "Name", "Creatures"],
+      [1, "Mammals",  [1, 3]],
+      [2, "Reptilia", [4]],
+    ])
+
+    self.remove_record("Creatures", 4)
+    self.assertTableData("Class", data=[
+      ["id", "Name", "Creatures"],
+      [1, "Mammals",  [1, 3]],
+      [2, "Reptilia", None]
+    ])
+
+
+  def test_contains(self):
+    self.load_sample(self.sample)
+    self.add_column('Class', 'ContainsInt', type='Any', isFormula=True,
+        formula="2 in $Creatures")
+    self.add_column('Class', 'ContainsRec', type='Any', isFormula=True,
+        formula="Creatures.lookupOne(Name='Chicken') in $Creatures")
+    self.add_column('Class', 'ContainsWrong', type='Any', isFormula=True,
+        formula="Class.lookupOne(Name='Reptilia') in $Creatures")
+
+    self.assertTableData("Class", data=[
+      ["id", "Name", "Creatures", "ContainsInt", "ContainsRec", "ContainsWrong"],
+      [1, "Mammals",  [1, 3],     False,          False,        False],
+      [2, "Reptilia", [2, 4],     True,           True,         False]
+    ])
+
+
+  def test_equals(self):
+    self.load_sample(self.sample)
+    self.add_column('Class', 'Lookup', type='RefList:Creatures', isFormula=True,
+        formula="Creatures.lookupRecords(Class=$id)")
+    self.add_column('Class', 'Equal', type='Any', isFormula=True,
+        formula="$Lookup == $Creatures")
+
+    self.assertTableData("Class", data=[
+      ["id", "Name", "Creatures", "Lookup", "Equal"],
+      [1, "Mammals",  [1, 3],     [1, 3],   True],
+      [2, "Reptilia", [2, 4],     [2, 4],   True],
+    ])

sandbox/grist/test_trigger_formulas.py

@@ -0,0 +1,524 @@
+import copy
+import time
+import logger
+import objtypes
+import testutil
+import test_engine
+from schema import RecalcWhen
+
+# pylint: disable=line-too-long
+
+log = logger.Logger(__name__, logger.INFO)
+
+attr_error = objtypes.RaisedException(AttributeError())
+
+class TestTriggerFormulas(test_engine.EngineTestCase):
+  col = testutil.col_schema_row
+  sample_desc = {
+    "SCHEMA": [
+      [1, "Creatures", [
+        col(1, "Name",       "Text",       False),
+        col(2, "Ocean",      "Ref:Oceans", False),
+        col(3, "OceanName",  "Text",       True,  "$Ocean.Name"),
+        col(4, "BossDef",    "Text",       False, "$Ocean.Head"),
+        col(5, "BossNvr",    "Text",       False, "$Ocean.Head", recalcWhen=RecalcWhen.NEVER),
+        col(6, "BossUpd",    "Text",       False, "$Ocean.Head", recalcDeps=[2]),
+        col(7, "BossAll",    "Text",       False, "$Ocean.Head", recalcWhen=RecalcWhen.MANUAL_UPDATES),
+      ]],
+      [2, "Oceans", [
+        col(11, "Name",     "Text",        False),
+        col(12, "Head",     "Text",        False)
+      ]],
+    ],
+    "DATA": {
+      "Creatures": [
+        ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll"],
+        [1,   "Dolphin", 2,       "Arthur",  "Arthur",  "Arthur",  "Arthur"],
+      ],
+      "Oceans": [
+        ["id",  "Name",     "Head"],
+        [1,     "Pacific",    "Watatsumi"],
+        [2,     "Atlantic",   "Poseidon"],
+        [3,     "Indian",     "Neptune"],
+        [4,     "Arctic",     "Poseidon"],
+      ],
+    }
+  }
+  sample = testutil.parse_test_sample(sample_desc)
+
+  def test_no_recalc_on_load(self):
+    # Trigger formulas don't affect data that's loaded.
+    self.load_sample(self.sample)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Dolphin", 2,       "Arthur",  "Arthur",  "Arthur",  "Arthur",  "Atlantic" ],
+    ])
+
+  def test_recalc_on_new_records(self):
+    # Trigger formulas affect new records.
+    self.load_sample(self.sample)
+    self.add_record("Creatures", Name="Shark", Ocean=2)
+    self.add_record("Creatures", Name="Squid", Ocean=1)
+
+    # Check that BossNvr ("never") wasn't affected by the default formula, but the rest were.
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",   "BossAll",   "OceanName"],
+      [1,   "Dolphin", 2,       "Arthur",    "Arthur",  "Arthur",    "Arthur",    "Atlantic" ],
+      [2,   "Shark",   2,       "Poseidon",  "",        "Poseidon",  "Poseidon",  "Atlantic" ],
+      [3,   "Squid",   1,       "Watatsumi", "",        "Watatsumi", "Watatsumi", "Pacific"  ],
+    ])
+
+  def test_no_recalc_on_noop_change(self):
+    # A no-op change shouldn't trigger any updates.
+    self.load_sample(self.sample)
+    self.update_record("Creatures", 1, Ocean=2)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Dolphin", 2,       "Arthur",  "Arthur",  "Arthur",  "Arthur",  "Atlantic" ],
+    ])
+
+  def test_recalc_on_update(self):
+    # Changes should trigger recalc of certain trigger formulas.
+    self.load_sample(self.sample)
+    self.add_record("Creatures", Name="Shark", Ocean=2)
+    self.add_record("Creatures", Name="Squid", Ocean=1)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",   "BossAll",   "OceanName"],
+      [1,   "Dolphin", 2,       "Arthur",    "Arthur",  "Arthur",    "Arthur",    "Atlantic" ],
+      [2,   "Shark",   2,       "Poseidon",  "",        "Poseidon",  "Poseidon",  "Atlantic" ],
+      [3,   "Squid",   1,       "Watatsumi", "",        "Watatsumi", "Watatsumi", "Pacific"  ],
+    ])
+    self.update_records("Creatures", ["id", "Ocean"], [
+      [1, 3],   # Ocean for 1: Atlantic -> Indian
+      [3, 4],   # Ocean for 3: Pacific -> Arctic
+    ])
+    # Only BossUpd and BossAll columns should be affected, not BossDef or BossNvr
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",   "BossAll",   "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",    "Arthur",  "Neptune",   "Neptune",    "Indian"  ],
+      [2,   "Shark",   2,       "Poseidon",  "",        "Poseidon",  "Poseidon",   "Atlantic"],
+      [3,   "Squid",   4,       "Watatsumi", "",        "Poseidon",  "Poseidon",   "Arctic"  ],
+    ])
+
+  def test_recalc_with_direct_update(self):
+    # Check that an update that changes both a dependency and the trigger-formula column itself
+    # respects the latter value.
+    self.load_sample(self.sample)
+
+    out_actions = self.update_record("Creatures", 1, Ocean=3, BossUpd="Bob")
+    self.assertTableData("Creatures", rows="subset", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",   "BossAll",   "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",    "Arthur",  "Bob",       "Neptune",    "Indian"  ],
+    ])
+    # Check that the needed recalcs are the only ones that happened.
+    self.assertPartialOutActions(out_actions, {
+      "calls": {"Creatures": {"BossAll": 1, "OceanName": 1}}
+    })
+
+    out_actions = self.update_record("Creatures", 1, Ocean=4, BossUpd="", BossAll="Chuck")
+    self.assertTableData("Creatures", rows="subset", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",   "BossAll",   "OceanName"],
+      [1,   "Dolphin", 4,       "Arthur",    "Arthur",  "",          "Chuck",     "Arctic"  ],
+    ])
+    # Check that the needed recalcs are the only ones that happened.
+    self.assertPartialOutActions(out_actions, {
+      "calls": {"Creatures": {"OceanName": 1}}
+    })
+
+  def test_no_recalc_on_reopen(self):
+    # Change that a reopen does not recalc at all.
+
+    # Load a sample with a few more rows. Only the one true formula should be calculated
+    sample_desc = copy.deepcopy(self.sample_desc)
+    sample_desc["DATA"]["Creatures"] = [
+      ["id","Name",    "Ocean", "BossDef",  "BossNvr", "BossUpd",  "BossAll" ],
+      [1,   "Dolphin", 2,       "Arthur",   "Arthur",  "Arthur",   "Arthur"  ],
+      [2,   "Shark",   2,       "",  "",               "Poseidon", "Poseidon"],
+      [3,   "Squid",   4,       "Watatsumi", "",       "Poseidon", ""        ],
+    ]
+    sample = testutil.parse_test_sample(sample_desc)
+
+    self.assertEqual(self.call_counts, {})
+    self.load_sample(sample)
+    self.assertEqual(self.call_counts, {'Creatures': {'OceanName': 3}})
+
+
+  def test_recalc_undo(self):
+    self.load_sample(self.sample)
+    data0 = [
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Dolphin", 2,       "Arthur",  "Arthur",  "Arthur",  "Arthur",  "Atlantic" ],
+    ]
+    self.assertTableData("Creatures", data=data0)
+
+    # Plain update
+    out_actions1 = self.update_record("Creatures", 1, Ocean=1)
+    data1 = [
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",   "BossAll",   "OceanName"],
+      [1,   "Dolphin", 1,       "Arthur",    "Arthur",  "Watatsumi", "Watatsumi", "Pacific"  ],
+    ]
+    self.assertTableData("Creatures", data=data1)
+    self.assertEqual(out_actions1.calls, {"Creatures": {"BossUpd": 1, "BossAll": 1, "OceanName": 1}})
+
+    # Update with a manual update to one of the trigger columns
+    out_actions2 = self.update_record("Creatures", 1, Ocean=3, BossUpd="Bob")
+    data2 = [
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",   "BossAll",   "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",    "Arthur",  "Bob",       "Neptune",   "Indian"  ],
+    ]
+    self.assertTableData("Creatures", rows="subset", data=data2)
+    self.assertEqual(out_actions2.calls, {"Creatures": {"BossAll": 1, "OceanName": 1}})
+
+    # Undo, one at a time. It should not cause recalc of trigger columns, because an undo sets
+    # those explicitly.
+    out_actions2_undo = self.apply_undo_actions(out_actions2.undo)
+    self.assertTableData("Creatures", data=data1)
+    self.assertEqual(out_actions2_undo.calls, {"Creatures": {"OceanName": 1}})
+
+    out_actions1_undo = self.apply_undo_actions(out_actions1.undo)
+    self.assertTableData("Creatures", data=data0)
+    self.assertEqual(out_actions1_undo.calls, {"Creatures": {"OceanName": 1}})
+
+
+  def test_recalc_triggers(self):
+    # A trigger that depends on some columns should not be triggered by other ones.
+    self.load_sample(self.sample)
+
+    # BossUpd and BossAll both depend on the "Ocean" column, so both get updated.
+    out_actions = self.update_record("Creatures", 1, Ocean=3)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",  "Arthur",  "Neptune", "Neptune", "Indian" ],
+    ])
+    self.assertEqual(out_actions.calls, {"Creatures": {"BossUpd": 1, "BossAll": 1, "OceanName": 1}})
+
+    # Undo, then check that a change that doesn't touch Ocean only triggers BossAll recalc.
+    self.apply_undo_actions(out_actions.undo)
+    out_actions = self.update_record("Creatures", 1, Name="Whale")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",  "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll",  "OceanName"],
+      [1,   "Whale", 2,       "Arthur",  "Arthur",  "Arthur",  "Poseidon", "Atlantic" ],
+    ])
+    self.assertEqual(out_actions.calls, {"Creatures": {"BossAll": 1}})
+
+
+  def test_recalc_trigger_changes(self):
+    # After changing a trigger formula dependencies, changes to the old dependency should no
+    # longer cause a recalc.
+    self.load_sample(self.sample)
+
+    # Change column BossUpd to depend on column Name rather than on column Ocean.
+    self.update_record("_grist_Tables_column", 6, recalcDeps=['L', 1])
+
+    # Make a change to Ocean. It should not cause an update to BossUpd, only BossAll.
+    out_actions = self.update_record("Creatures", 1, Ocean=3)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",  "Arthur",  "Arthur",  "Neptune", "Indian" ],
+    ])
+    self.assertEqual(out_actions.calls, {"Creatures": {"BossAll": 1, "OceanName": 1}})
+
+    # But changes to the new dependency should trigger recalc.
+    out_actions = self.update_record("Creatures", 1, Name="Whale")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",  "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Whale", 3,       "Arthur",  "Arthur",  "Neptune", "Neptune", "Indian" ],
+    ])
+    self.assertEqual(out_actions.calls, {"Creatures": {"BossUpd": 1, "BossAll": 1}})
+
+    # If dependencies are changed to empty, only new records should cause BossUpd recalc.
+    self.update_record("_grist_Tables_column", 6, recalcDeps=['L'])
+    out_actions = self.update_record("Creatures", 1, Name="Porpoise", Ocean=2)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",     "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll",  "OceanName"],
+      [1,   "Porpoise", 2,       "Arthur",  "Arthur",  "Neptune", "Poseidon", "Atlantic" ],
+    ])
+    self.assertEqual(out_actions.calls, {"Creatures": {"BossAll": 1, "OceanName": 1}})
+
+    out_actions = self.add_record("Creatures", None, Name="Manatee", Ocean=2)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",     "Ocean", "BossDef", "BossNvr", "BossUpd",  "BossAll",  "OceanName"],
+      [1,   "Porpoise", 2,       "Arthur",  "Arthur",  "Neptune",  "Poseidon", "Atlantic" ],
+      [2,   "Manatee",  2,       "Poseidon", "",       "Poseidon", "Poseidon", "Atlantic" ],
+    ])
+    self.assertEqual(out_actions.calls,
+        {"Creatures": {"BossDef": 1, "BossUpd": 1, "BossAll": 1, "OceanName": 1}})
+
+
+  def test_recalc_trigger_off(self):
+    # Change BossUpd dependency to never, and check that neither changes nor new records cause
+    # recalc.
+    self.load_sample(self.sample)
+    self.update_record("_grist_Tables_column", 6, recalcWhen=RecalcWhen.NEVER)
+
+    # Check a change
+    out_actions = self.update_record("Creatures", 1, Name="Whale", Ocean=3)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",  "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Whale", 3,       "Arthur",  "Arthur",  "Arthur",  "Neptune", "Indian" ],
+    ])
+    self.assertEqual(out_actions.calls, {"Creatures": {"BossAll": 1, "OceanName": 1}})
+
+    # Check a new record -- doesn't affect BossUpd any more.
+    out_actions = self.add_record("Creatures", None, Name="Manatee", Ocean=2)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",     "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll",  "OceanName"],
+      [1,   "Whale",    3,       "Arthur",  "Arthur",  "Arthur",  "Neptune",  "Indian" ],
+      [2,   "Manatee",  2,       "Poseidon", "",       "",        "Poseidon", "Atlantic" ],
+    ])
+    self.assertEqual(out_actions.calls,
+        {"Creatures": {"BossDef": 1, "BossAll": 1, "OceanName": 1}})
+
+
+  def test_renames(self):
+    # After renaming tables or columns, trigger formulas should still be triggered the same way.
+    self.load_sample(self.sample)
+
+    # Do some renamings: they shouldn't trigger updates to trigger formulas.
+    self.apply_user_action(["RenameColumn", "Creatures", "Ocean", "Sea"])
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Sea", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Dolphin", 2,     "Arthur",  "Arthur",  "Arthur",  "Arthur",  "Atlantic" ],
+    ])
+
+    self.apply_user_action(["RenameColumn", "Creatures", "BossUpd", "foo"])
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Sea", "BossDef", "BossNvr", "foo",     "BossAll", "OceanName"],
+      [1,   "Dolphin", 2,     "Arthur",  "Arthur",  "Arthur",  "Arthur",  "Atlantic" ],
+    ])
+
+    self.apply_user_action(["RenameTable", "Creatures", "Critters"])
+    self.assertTableData("Critters", data=[
+      ["id","Name",    "Sea", "BossDef", "BossNvr", "foo",     "BossAll", "OceanName"],
+      [1,   "Dolphin", 2,     "Arthur",  "Arthur",  "Arthur",  "Arthur",  "Atlantic" ],
+    ])
+
+    self.apply_user_action(["RenameColumn", "Critters", "BossAll", "bar"])
+    self.assertTableData("Critters", data=[
+      ["id","Name",    "Sea", "BossDef", "BossNvr", "foo",     "bar",     "OceanName"],
+      [1,   "Dolphin", 2,     "Arthur",  "Arthur",  "Arthur",  "Arthur",  "Atlantic" ],
+    ])
+
+    # After renames, correct trigger formulas continue getting triggered.
+    out_actions = self.update_record("Critters", 1, Sea=3)
+    self.assertTableData("Critters", data=[
+      ["id","Name",    "Sea",   "BossDef", "BossNvr", "foo",     "bar",     "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",  "Arthur",  "Neptune", "Neptune", "Indian" ],
+    ])
+    self.assertEqual(out_actions.calls, {"Critters": {"foo": 1, "bar": 1, "OceanName": 1}})
+
+    # After renames, changes shouldn't trigger unnecessary recalcs (foo, formerly BossUpd, should
+    # not be triggered by a change to Name).
+    out_actions = self.update_record("Critters", 1, Name="Whale")
+    self.assertTableData("Critters", data=[
+      ["id","Name",  "Sea",   "BossDef", "BossNvr", "foo",      "bar",     "OceanName"],
+      [1,   "Whale", 3,       "Arthur",  "Arthur",  "Neptune",  "Neptune", "Indian" ],
+    ])
+    self.assertEqual(out_actions.calls, {"Critters": {"bar": 1}})
+
+
+  def test_schema_changes(self):
+    # Schema changes like add/modify column should not cause trigger-formulas to recalculate.
+    self.load_sample(self.sample)
+
+    # Adding a column doesn't trigger recalcs.
+    out_actions = self.apply_user_action(["AddColumn", "Creatures", "Size", {"type": "Text", "isFormula": False}])
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName", "Size"],
+      [1,   "Dolphin", 2,       "Arthur",  "Arthur",  "Arthur",  "Arthur",  "Atlantic",  ""],
+    ])
+    self.assertEqual(out_actions.calls, {})
+
+    # Only BossAll should recalc since the record changed.
+    out_actions = self.update_record("Creatures", 1, Size="Big")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll",  "OceanName", "Size"],
+      [1,   "Dolphin", 2,       "Arthur",  "Arthur",  "Arthur",  "Poseidon", "Atlantic",  "Big"],
+    ])
+    self.assertEqual(out_actions.calls, {"Creatures": {"BossAll": 1}})
+
+    # New records trigger recalc as usual.
+    out_actions = self.add_record("Creatures", None, Name="Manatee", Ocean=2)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",  "BossNvr", "BossUpd",  "BossAll",  "OceanName", "Size"],
+      [1,   "Dolphin", 2,       "Arthur",   "Arthur",  "Arthur",   "Poseidon", "Atlantic",  "Big"],
+      [2,   "Manatee", 2,       "Poseidon", "",        "Poseidon", "Poseidon", "Atlantic",  ""],
+    ])
+
+    # ModifyColumn doesn't trigger recalcs.
+    out_actions = self.apply_user_action(["ModifyColumn", "Creatures", "Size", {type: 'Numeric'}])
+    self.assertEqual(out_actions.calls, {})
+
+
+  def test_changing_trigger_formula(self):
+    self.load_sample(self.sample)
+
+    # Modifying trigger formula doesn't trigger recalc.
+    out_actions = self.apply_user_action(["ModifyColumn", "Creatures", "BossAll", {"formula": 'UPPER($Ocean.Head)'}])
+    self.assertEqual(out_actions.calls, {})
+
+    # But when it runs, recalc uses the new formula.
+    out_actions = self.update_record("Creatures", 1, Name="Whale")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",  "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll",  "OceanName"],
+      [1,   "Whale", 2,       "Arthur",  "Arthur",  "Arthur",  "POSEIDON", "Atlantic" ],
+    ])
+
+
+  def test_remove_dependency(self):
+    # Remove a dependency column, and check that recalcDeps list is updated.
+    self.load_sample(self.sample)
+
+    def get_recalc_deps(col_ref):
+      data = self.engine.fetch_table('_grist_Tables_column', col_ref, query={'id': [col_ref]})
+      return data.columns['recalcDeps'][0]
+
+    self.assertEqual(get_recalc_deps(6), [2])
+
+    # Add another dependency, so that we can test partial removal.
+    self.update_record("_grist_Tables_column", 6, recalcDeps=['L', 2, 3])
+    self.assertEqual(get_recalc_deps(6), [2, 3])
+
+    # Remove a column that it's a Dependency of BossUpd
+    self.apply_user_action(["RemoveColumn", "Creatures", "Ocean"])
+    self.assertEqual(get_recalc_deps(6), [3])
+    self.apply_user_action(["RemoveColumn", "Creatures", "OceanName"])
+    self.assertEqual(get_recalc_deps(6), None)
+
+    # None of these operations should have changed trigger-formula columns.
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "BossDef", "BossNvr", "BossUpd", "BossAll"],
+      [1,   "Dolphin", "Arthur",  "Arthur",  "Arthur",  "Arthur" ],
+    ])
+
+    # Check that it still responds to suitable triggers.
+    # Make a change to some other column. BossUpd doesn't get updated.
+    out_actions = self.update_record("Creatures", 1, Name="Whale")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",  "BossDef", "BossNvr", "BossUpd", "BossAll" ],
+      [1,   "Whale", "Arthur",  "Arthur",  "Arthur",  attr_error],
+    ])
+
+    # Add a record. BossUpd's formula still runs, though with an error.
+    out_actions = self.add_record("Creatures", None, Name="Manatee")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "BossDef",  "BossNvr", "BossUpd",  "BossAll" ],
+      [1,   "Whale",   "Arthur",   "Arthur",  "Arthur",   attr_error],
+      [2,   "Manatee", attr_error, "",        attr_error, attr_error],
+    ])
+
+
+  def test_no_trigger_by_formulas(self):
+    # A column that depends on any record update ("allupdates") should not be affected by formula
+    # recalculations.
+    self.load_sample(self.sample)
+
+    # Name of Ocean affects a formula column; Head affects calculation; neither triggers recalc.
+    self.update_record('Oceans', 2, Head="POSEIDON", Name="ATLANTIC")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Dolphin", 2,       "Arthur",  "Arthur",  "Arthur",  "Arthur",  "ATLANTIC" ],
+    ])
+    self.add_record("Creatures", None, Name="Manatee", Ocean=2)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll", "OceanName"],
+      [1,   "Dolphin", 2,       "Arthur",  "Arthur",  "Arthur",  "Arthur",  "ATLANTIC" ],
+      [2,   "Manatee", 2,       "POSEIDON",  "",  "POSEIDON",  "POSEIDON",  "ATLANTIC" ],
+    ])
+
+    # On the other hand, an explicit dependency on a formula column WILL be triggered.
+    self.update_record("_grist_Tables_column", 6, recalcDeps=['L', 2, 3])
+    self.update_record('Oceans', 2, Name="atlantic")
+
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef", "BossNvr", "BossUpd", "BossAll",    "OceanName"],
+      [1,   "Dolphin", 2,       "Arthur",  "Arthur",  "POSEIDON",  "Arthur",   "atlantic" ],
+      [2,   "Manatee", 2,       "POSEIDON",  "",      "POSEIDON",  "POSEIDON", "atlantic" ],
+    ])
+
+
+  def test_no_auto_dependencies(self):
+    # Evaluating a trigger formula should not create dependencies on cells used during
+    # evaluation.
+    self.load_sample(self.sample)
+    self.update_record("Creatures", 1, Ocean=3)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",   "BossAll", "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",    "Arthur",  "Neptune",   "Neptune", "Indian"  ],
+    ])
+    # Update a value that trigger-cells used during calculation; it should not cause a recalc.
+    self.update_record('Oceans', 3, Head="NEPTUNE")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",   "BossAll", "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",    "Arthur",  "Neptune",   "Neptune", "Indian"  ],
+    ])
+
+
+  def test_self_trigger(self):
+    # A trigger formula may be triggered by changes to the column itself.
+    # Check that it gets recalculated.
+    self.load_sample(self.sample)
+    # Set BossUpd column to depend on Ocean and itself.
+    # Append something to ensure we are testing a case without a fixed point, to ensure
+    # that doesn't cause an infinite update loop.
+    self.update_record('_grist_Tables_column', 6, recalcDeps=['L', 2, 6],
+        formula="UPPER(value or $Ocean.Head) + '+'")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",  "BossAll", "OceanName"],
+      [1,   "Dolphin", 2,       "Arthur",    "Arthur",  "Arthur",   "Arthur",  "Atlantic"  ],
+    ])
+
+    self.update_record('Creatures', 1, Ocean=3)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",  "BossAll", "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",    "Arthur",  "ARTHUR+",   "Neptune", "Indian"  ],
+    ])
+    self.update_record('Creatures', 1, BossUpd="None")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",  "BossAll", "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",    "Arthur",  "NONE+",     "Neptune", "Indian"  ],
+    ])
+    self.update_record('Creatures', 1, BossUpd="")
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",  "BossAll", "OceanName"],
+      [1,   "Dolphin", 3,       "Arthur",    "Arthur",  "NEPTUNE+",  "Neptune", "Indian"  ],
+    ])
+
+  def test_last_update_recipe(self):
+    # Use a formula to store time of last-update. Check that it works as expected.
+    # Check that times don't update on reload.
+    self.load_sample(self.sample)
+    self.add_column('Creatures', 'LastChange',
+      type='DateTime:UTC', isFormula=False, formula="NOW()", recalcWhen=RecalcWhen.MANUAL_UPDATES)
+
+    # To compare times, use actual times after checking approximately.
+    now = time.time()
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd", "BossAll", "OceanName", "LastChange"],
+      [1,   "Dolphin", 2,       "Arthur",    "Arthur",  "Arthur",  "Arthur",  "Atlantic",  None],
+    ])
+
+    self.add_record("Creatures", None, Name="Manatee", Ocean=2)
+    self.update_record("Creatures", 1, Ocean=3)
+
+    now = time.time()
+    [time1, time2] = self.engine.fetch_table('Creatures').columns['LastChange']
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",  "BossAll",  "OceanName", "LastChange"],
+      [1,   "Dolphin", 3,       "Arthur",    "Arthur",  "Neptune",  "Neptune",  "Indian",    time1],
+      [2,   "Manatee", 2,       "Poseidon",  "",        "Poseidon", "Poseidon", "Atlantic",  time2],
+    ])
+    self.assertLessEqual(abs(time1 - now), 1)
+    self.assertLessEqual(abs(time2 - now), 1)
+
+    # An indirect change doesn't affect the time, but a direct change does.
+    self.update_record("Oceans", 2, Name="ATLANTIC")
+    self.update_record("Creatures", 1, Name="Whale")
+    [time3, time4] = self.engine.fetch_table('Creatures').columns['LastChange']
+    self.assertGreater(time3, time1)
+    self.assertEqual(time4, time2)
+    self.assertTableData("Creatures", data=[
+      ["id","Name",    "Ocean", "BossDef",   "BossNvr", "BossUpd",  "BossAll",  "OceanName", "LastChange"],
+      [1,   "Whale",   3,       "Arthur",    "Arthur",  "Neptune",  "Neptune",  "Indian",    time3],
+      [2,   "Manatee", 2,       "Poseidon",  "",        "Poseidon", "Poseidon", "ATLANTIC",  time2],
+    ])

sandbox/grist/testutil.py

@@ -99,8 +99,8 @@ def parse_test_sample(obj, samples={}):
       '_grist_Tables_column': table_data_from_rows(
         '_grist_Tables_column',
         ("parentId", "parentPos", "id", "colId", "type", "isFormula",
-         "formula", "label", "widgetOptions"),
+         "formula", "label", "widgetOptions", "recalcWhen", "recalcDeps"),
-        [[table_row_id, i+1] + e for (table_row_id, _, entries) in raw_schema
+        [[table_row_id, i+1] + col_schema_row(*e) for (table_row_id, _, entries) in raw_schema
          for (i, e) in enumerate(entries)])
     }
 
@@ -109,6 +109,14 @@ def parse_test_sample(obj, samples={}):
   return {"SCHEMA": schema, "DATA": data}
 
 
+def col_schema_row(id_, colId, type_, isFormula, formula="",
+                   label="", widgetOptions="", recalcWhen=0, recalcDeps=None):
+  """
+  Helper to specify columns in test SCHEMA descriptions, to allow omitting some column properties.
+  """
+  return [id_, colId, type_, isFormula, formula, label, widgetOptions, recalcWhen, recalcDeps]
+
+
 def replace_nans(data):
   """
   Convert all NaNs and Infinities in the data to descriptive strings, since they cannot be

sandbox/grist/useractions.py

@@ -14,6 +14,7 @@ import column
 import identifiers
 from objtypes import strict_equal
 import schema
+from schema import RecalcWhen
 import summary
 import import_actions
 import repl
@@ -331,10 +332,18 @@ class UserActions(object):
 
     self._do_doc_action(action)
 
-    # Invalidate new records, including the omitted columns that may have default formulas,
+    # Invalidate new records, including the columns that may have default formulas (trigger
-    # in order to get dynamically-computed default values.
+    # formulas set to recalculate on new records), to get dynamically-computed default values.
-    omitted_cols = six.viewkeys(table.all_columns) - six.viewkeys(column_values)
+    recalc_cols = set()
-    self._engine.invalidate_records(table_id, filled_row_ids, data_cols_to_recompute=omitted_cols)
+    for col_id in table.all_columns:
+      if col_id in column_values:
+        continue
+      col_rec = self._docmodel.columns.lookupOne(tableId=table_id, colId=col_id)
+      if col_rec.recalcWhen == RecalcWhen.NEVER:
+        continue
+      recalc_cols.add(col_id)
+
+    self._engine.invalidate_records(table_id, filled_row_ids, data_cols_to_recompute=recalc_cols)
 
     return filled_row_ids
 
@@ -363,6 +372,27 @@ class UserActions(object):
     # Finally, update the record
     self._do_doc_action(action)
 
+    # Invalidate trigger-formula columns affected by this update.
+    table = self._engine.tables[table_id]
+    column_values = action[2]
+    if column_values:     # Only if this is a non-trivial update.
+      for col_id, col_obj in table.all_columns.iteritems():
+        if col_obj.is_formula() or not col_obj.has_formula():
+          continue
+        col_rec = self._docmodel.columns.lookupOne(tableId=table_id, colId=col_id)
+
+        # Schedule for recalculation those trigger-formulas that depend on any manual update.
+        if col_rec.recalcWhen == RecalcWhen.MANUAL_UPDATES:
+          self._engine.invalidate_column(col_obj, row_ids, recompute_data_col=True)
+
+        # When we have an explicit value for a trigger-formula, the logic in docactions.py
+        # normally prevents recalculation so that the explicit value would stay (it is also
+        # important for undos). For a data-cleaning column (one that depends on itself), a manual
+        # change *should* trigger recalculation, so we un-prevent it here.
+        if col_id in column_values and col_rec.recalcOnChangesToSelf:
+          self._engine.prevent_recalc(col_obj.node, row_ids, should_prevent=False)
+
+
   # Helper to perform doBulkUpdateRecord using record update value pairs. This saves
   #  the steps of separating the value pairs into row ids and column values.
   # The record_values_pairs should be given as a list of tuples, the first element of each
@@ -556,6 +586,10 @@ class UserActions(object):
     self._docmodel.update([f for c in type_changed for f in c.viewFields],
                           widgetOptions='', displayCol=0)
 
+    # If the column update changes its trigger-formula conditions, rebuild dependencies.
+    if any(("recalcWhen" in values or "recalcDeps" in values) for c, values in update_pairs):
+      self._engine.trigger_columns_changed()
+
     self.doBulkUpdateFromPairs(table_id, update_pairs)
     make_acl_updates()
 
@@ -727,14 +761,16 @@ class UserActions(object):
     self._do_doc_action(actions.BulkRemoveRecord(table_id, row_ids))
 
     # Also remove any references to this row from other tables.
+    row_id_set = set(row_ids)
     for ref_col in table._back_references:
-      if ref_col.is_formula():
+      if ref_col.is_formula() or not isinstance(ref_col, column.BaseReferenceColumn):
         continue
-      affected_rows = sorted(ref_col._relation.get_affected_rows(row_ids))
+      updates = ref_col.get_updates_for_removed_target_rows(row_id_set)
-      if affected_rows:
+      if updates:
-        self._do_doc_action(actions.BulkUpdateRecord(ref_col.table_id, affected_rows, {
+        self._do_doc_action(actions.BulkUpdateRecord(ref_col.table_id,
-          ref_col.col_id: [ref_col.getdefault() for row_id in affected_rows]
+          [row_id for (row_id, value) in updates],
-        }))
+          { ref_col.col_id: [value for (row_id, value) in updates] }
+        ))
 
   @useraction
   def RemoveRecord(self, table_id, row_id):
@@ -986,6 +1022,10 @@ class UserActions(object):
       'widgetOptions': col_info.get('widgetOptions', ''),
       'label': col_info.get('label', col_id),
     })
+    if 'recalcWhen' in col_info:
+      values['recalcWhen'] = col_info['recalcWhen']
+    if 'recalcDeps' in col_info:
+      values['recalcDeps'] = col_info['recalcDeps']
     visible_col = col_info.get('visibleCol', 0)
     if visible_col:
       values['visibleCol'] = visible_col

sandbox/grist/usercode.py

@@ -53,7 +53,7 @@ class Address:
   city = grist.Text()
   state = grist.Text()
 
-  def _default_country(rec, table):
+  def _default_country(rec, table, value):
     return 'US'
   country = grist.Text()