(core) Update ACL resources/rules when tables/columns get renamed

Summary:
- Placed rule-updating functions in acl.py.
- Reset UI when rules update externally, or alert the user to reset if there
  are pending local changes.
- Removed some unused and distracting bits from client-side DocModel.

A few improvements related to poor error handling:
- In case of missing DocActions (tickled by broken ACL rule handling), don't
  add to confusion by attempting to process bad actions
- In case of missing attributes in ACL formulas, return undefined rather than
  fail; the latter creates more problems.
- In case in invalid rules, fail rather than skip; this feels more correct now
  that we have error checking and recovery option, and helps avoid invalid rules.
- Prevent saving invalid rules with an empty ACL formula.
- Fix bug with rule positions.

Test Plan: Added a python and browser test for table/column renames.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2698

sandbox/grist/acl.py

@@ -2,7 +2,15 @@
 # It now retains only the minimum needed to keep new documents openable by old code,
 # and to produce the ActionBundles expected by other code.
 
+import json
+
+from acl_formula import parse_acl_grist_entities
 import action_obj
+import logger
+import textbuilder
+
+log = logger.Logger(__name__, logger.INFO)
+
 
 class Permissions(object):
   # Permission types and their combination are represented as bits of a single integer.
@@ -36,3 +44,93 @@ def acl_read_split(action_group):
   bundle.undo.extend((0, da) for da in action_group.undo)
   bundle.retValues = action_group.retValues
   return bundle
+
+
+def prepare_acl_table_renames(docmodel, useractions, table_renames_dict):
+  """
+  Given a dict of table renames of the form {table_id: new_table_id}, returns a callback
+  that will apply updates to the affected ACL rules and resources.
+  """
+  # If there are ACLResources that refer to the renamed table, prepare updates for those.
+  resource_updates = []
+  for resource_rec in docmodel.aclResources.all:
+    if resource_rec.tableId in table_renames_dict:
+      resource_updates.append((resource_rec, {'tableId': table_renames_dict[resource_rec.tableId]}))
+
+  # Collect updates for any ACLRules with UserAttributes that refer to the renamed table.
+  rule_updates = []
+  for rule_rec in docmodel.aclRules.all:
+    if rule_rec.userAttributes:
+      try:
+        rule_info = json.loads(rule_rec.userAttributes)
+        if rule_info.get("tableId") in table_renames_dict:
+          rule_info["tableId"] = table_renames_dict[rule_info.get("tableId")]
+          rule_updates.append((rule_rec, {'userAttributes': json.dumps(rule_info)}))
+      except Exception, e:
+        log.warn("Error examining aclRule: %s" % (e,))
+
+  def do_renames():
+    useractions.doBulkUpdateFromPairs('_grist_ACLResources', resource_updates)
+    useractions.doBulkUpdateFromPairs('_grist_ACLRules', rule_updates)
+  return do_renames
+
+
+def prepare_acl_col_renames(docmodel, useractions, col_renames_dict):
+  """
+  Given a dict of column renames of the form {(table_id, col_id): new_col_id}, returns a callback
+  that will apply updates to the affected ACL rules and resources.
+  """
+  # Collect updates for ACLResources that refer to the renamed columns.
+  resource_updates = []
+  for resource_rec in docmodel.aclResources.all:
+    t = resource_rec.tableId
+    if resource_rec.colIds and resource_rec.colIds != '*':
+      new_col_ids = ','.join((col_renames_dict.get((t, c)) or c)
+                             for c in resource_rec.colIds.split(','))
+      if new_col_ids != resource_rec.colIds:
+        resource_updates.append((resource_rec, {'colIds': new_col_ids}))
+
+  # Collect updates for any ACLRules with UserAttributes that refer to the renamed column.
+  rule_updates = []
+  user_attr_tables = {}   # Maps name of user attribute to its lookup table
+  for rule_rec in docmodel.aclRules.all:
+    if rule_rec.userAttributes:
+      try:
+        rule_info = json.loads(rule_rec.userAttributes)
+        user_attr_tables[rule_info.get('name')] = rule_info.get('tableId')
+        new_col_id = col_renames_dict.get((rule_info.get("tableId"), rule_info.get("lookupColId")))
+        if new_col_id:
+          rule_info["lookupColId"] = new_col_id
+          rule_updates.append((rule_rec, {'userAttributes': json.dumps(rule_info)}))
+      except Exception, e:
+        log.warn("Error examining aclRule: %s" % (e,))
+
+  # Go through again checking if anything in ACL formulas is affected by the rename.
+  for rule_rec in docmodel.aclRules.all:
+    if rule_rec.aclFormula:
+      # Positions are obtained from unicode version of formulas, so that's what we must patch
+      formula = rule_rec.aclFormula.decode('utf8')
+      patches = []
+
+      for entity in parse_acl_grist_entities(rule_rec.aclFormula):
+        if entity.type == 'recCol':
+          table_id = docmodel.aclResources.table.get_record(int(rule_rec.resource)).tableId
+        elif entity.type == 'userAttrCol':
+          table_id = user_attr_tables.get(entity.extra)
+        else:
+          continue
+        col_id = entity.name
+        new_col_id = col_renames_dict.get((table_id, col_id))
+        if not new_col_id:
+          continue
+        patch = textbuilder.make_patch(
+            formula, entity.start_pos, entity.start_pos + len(entity.name), new_col_id)
+        patches.append(patch)
+
+      replacer = textbuilder.Replacer(textbuilder.Text(formula), patches)
+      rule_updates.append((rule_rec, {'aclFormula': replacer.get_text().encode('utf8')}))
+
+  def do_renames():
+    useractions.doBulkUpdateFromPairs('_grist_ACLResources', resource_updates)
+    useractions.doBulkUpdateFromPairs('_grist_ACLRules', rule_updates)
+  return do_renames

sandbox/grist/acl_formula.py

@@ -1,5 +1,8 @@
 import ast
 import json
+from collections import namedtuple
+
+import asttokens
 
 def parse_acl_formula(acl_formula):
   """
@@ -35,6 +38,27 @@ def parse_acl_formula_json(acl_formula):
   return json.dumps(parse_acl_formula(acl_formula)) if acl_formula else ""
 
 
+# Entities encountered in ACL formulas, which may get renamed.
+#   type: 'recCol'|'userAttr'|'userAttrCol',
+#   start_pos: number,        # start position of the token in the code.
+#   name: string,             # the name that may be updated by a rename.
+#   extra: string|None,       # name of userAttr in case of userAttrCol; otherwise None.
+NamedEntity = namedtuple('NamedEntity', ('type', 'start_pos', 'name', 'extra'))
+
+def parse_acl_grist_entities(acl_formula):
+  """
+  Parse the ACL formula collecting any entities that may be subject to renaming. Returns a
+  NamedEntity list.
+  """
+  try:
+    atok = asttokens.ASTTokens(acl_formula, tree=ast.parse(acl_formula, mode='eval'))
+    converter = _EntityCollector()
+    converter.visit(atok.tree)
+    return converter.entities
+  except SyntaxError as err:
+    return []
+
+
 named_constants = {
   'True': True,
   'False': False,
@@ -89,3 +113,25 @@ class _TreeConverter(ast.NodeVisitor):
 
   def generic_visit(self, node):
     raise ValueError("Unsupported syntax at %s:%s" % (node.lineno, node.col_offset + 1))
+
+
+class _EntityCollector(_TreeConverter):
+  def __init__(self):
+    self.entities = []    # NamedEntity list
+
+  def visit_Attribute(self, node):
+    parent = self.visit(node.value)
+
+    # We recognize a couple of specific patterns for entities that may be affected by renames.
+    if parent == ['Name', 'rec']:
+      # rec.COL refers to the column from the table that the rule is on.
+      self.entities.append(NamedEntity('recCol', node.last_token.startpos, node.attr, None))
+    if parent == ['Name', 'user']:
+      # user.ATTR is a user attribute.
+      self.entities.append(NamedEntity('userAttr', node.last_token.startpos, node.attr, None))
+    elif parent[0] == 'Attr' and parent[1] == ['Name', 'user']:
+      # user.ATTR.COL is a column from the lookup table of the UserAttribute ATTR.
+      self.entities.append(
+          NamedEntity('userAttrCol', node.last_token.startpos, node.attr, parent[2]))
+
+    return ["Attr", parent, node.attr]

sandbox/grist/docmodel.py

@@ -147,6 +147,8 @@ class DocModel(object):
     self.repl_hist               = self._prep_table("_grist_REPL_Hist")
     self.attachments             = self._prep_table("_grist_Attachments")
     self.pages                   = self._prep_table("_grist_Pages")
+    self.aclResources            = self._prep_table("_grist_ACLResources")
+    self.aclRules                = self._prep_table("_grist_ACLRules")
 
   def _prep_table(self, name):
     """

sandbox/grist/test_acl_renames.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+
+import json
+
+import test_engine
+import testsamples
+import useractions
+
+user_attr1 = {
+    'name': 'School',
+    'charId': 'Email',
+    'tableId': 'Schools',
+    'lookupColId': 'LiasonEmail',
+}
+
+class TestACLRenames(test_engine.EngineTestCase):
+
+  def setUp(self):
+    super(TestACLRenames, self).setUp()
+
+    self.load_sample(testsamples.sample_students)
+
+    # Add column to Schools to use with User Attribute.
+    self.engine.apply_user_actions([useractions.from_repr(ua) for ua in (
+      ['AddColumn', 'Schools', 'LiasonEmail', {'type': 'Text'}],
+      ['AddRecord', '_grist_ACLResources', -1, {'tableId': '*', 'colIds': '*'}],
+      ['AddRecord', '_grist_ACLRules', None, {
+        'resource': -1,
+        'userAttributes': json.dumps(user_attr1),
+      }],
+      ['AddRecord', '_grist_ACLResources', -2, {
+        'tableId': 'Students', 'colIds': 'firstName,lastName'
+      }],
+      ['AddRecord', '_grist_ACLResources', -3, {
+        'tableId': 'Students', 'colIds': '*'
+      }],
+      ['AddRecord', '_grist_ACLRules', None, {
+        'resource': -2,
+        # Include comments and unicode to check that renaming respects all that.
+        'aclFormula': '( rec.schoolName !=  # ünîcødé comment\n  user.School.name)',
+        'permissionsText': 'none',
+      }],
+      ['AddRecord', '_grist_ACLRules', None, {
+        'resource': -3,
+        'permissionsText': 'all'
+      }],
+    )])
+
+    # Here's what we expect to be in the ACL tables (for reference in tests below).
+    self.assertTableData('_grist_ACLResources', cols="subset", data=[
+      ['id',  'tableId',  'colIds'],
+      [1,     '*',        '*'],
+      [2,     'Students', 'firstName,lastName'],
+      [3,     'Students', '*'],
+    ])
+    self.assertTableData('_grist_ACLRules', cols="subset", data=[
+      ['id',  'resource', 'aclFormula', 'permissionsText', 'userAttributes'],
+      [1,     1,          '',           '',                json.dumps(user_attr1)],
+      [2,     2,  '( rec.schoolName !=  # ünîcødé comment\n  user.School.name)', 'none', ''],
+      [3,     3,          '',           'all',              ''],
+    ])
+
+  def test_acl_table_renames(self):
+    # Rename some tables.
+    self.apply_user_action(['RenameTable', 'Students', 'Estudiantes'])
+    self.apply_user_action(['RenameTable', 'Schools', 'Escuelas'])
+
+    user_attr1_renamed = dict(user_attr1, tableId='Escuelas')
+
+    # Check the result of both renames.
+    self.assertTableData('_grist_ACLResources', cols="subset", data=[
+      ['id', 'tableId', 'colIds'],
+      [1,     '*',        '*'],
+      [2,     'Estudiantes', 'firstName,lastName'],
+      [3,     'Estudiantes', '*'],
+    ])
+    self.assertTableData('_grist_ACLRules', cols="subset", data=[
+      ['id',  'resource', 'aclFormula', 'permissionsText', 'userAttributes'],
+      [1,     1,          '',           '',                json.dumps(user_attr1_renamed)],
+      [2,     2,  '( rec.schoolName !=  # ünîcødé comment\n  user.School.name)', 'none', ''],
+      [3,     3,          '',           'all',              ''],
+    ])
+
+  def test_acl_column_renames(self):
+    # Rename some columns.
+    self.apply_user_action(['RenameColumn', 'Students', 'lastName', 'Family_Name'])
+    self.apply_user_action(['RenameColumn', 'Schools', 'name', 'schoolName'])
+    self.apply_user_action(['RenameColumn', 'Students', 'schoolName', 'escuela'])
+    self.apply_user_action(['RenameColumn', 'Schools', 'LiasonEmail', 'AdminEmail'])
+
+    user_attr1_renamed = dict(user_attr1, lookupColId='AdminEmail')
+
+    # Check the result of both renames.
+    self.assertTableData('_grist_ACLResources', cols="subset", data=[
+      ['id', 'tableId', 'colIds'],
+      [1,     '*',        '*'],
+      [2,     'Students', 'firstName,Family_Name'],
+      [3,     'Students', '*'],
+    ])
+    self.assertTableData('_grist_ACLRules', cols="subset", data=[
+      ['id',  'resource', 'aclFormula', 'permissionsText', 'userAttributes'],
+      [1,     1,          '',           '',                json.dumps(user_attr1_renamed)],
+      [2,     2,  '( rec.escuela !=  # ünîcødé comment\n  user.School.schoolName)', 'none', ''],
+      [3,     3,          '',           'all',              ''],
+    ])
+
+  def test_multiple_renames(self):
+    # Combine several renames into one bundle.
+    self.engine.apply_user_actions([useractions.from_repr(ua) for ua in (
+      ['RenameColumn', 'Students', 'firstName', 'Given_Name'],
+      ['RenameColumn', 'Students', 'lastName', 'Family_Name'],
+      ['RenameTable', 'Students', 'Students2'],
+      ['RenameColumn', 'Students2', 'schoolName', 'escuela'],
+      ['RenameColumn', 'Schools', 'name', 'schoolName'],
+    )])
+    self.assertTableData('_grist_ACLResources', cols="subset", data=[
+      ['id', 'tableId',     'colIds'],
+      [1,     '*',          '*'],
+      [2,     'Students2',  'Given_Name,Family_Name'],
+      [3,     'Students2',  '*'],
+    ])
+    self.assertTableData('_grist_ACLRules', cols="subset", data=[
+      ['id',  'resource', 'aclFormula', 'permissionsText', 'userAttributes'],
+      [1,     1,          '',           '',                json.dumps(user_attr1)],
+      [2,     2,  '( rec.escuela !=  # ünîcødé comment\n  user.School.schoolName)', 'none', ''],
+      [3,     3,          '',           'all',              ''],
+    ])

sandbox/grist/useractions.py

@@ -465,6 +465,8 @@ class UserActions(object):
       if 'type' in values:
         self.doModifyColumn(col.tableId, col.colId, {'type': 'Int'})
 
+    make_acl_updates = acl.prepare_acl_table_renames(self._docmodel, self, table_renames)
+
     # Collect all the table renames, and do the actual schema actions to apply them.
     for tbl, values in update_pairs:
       if has_diff_value(values, 'tableId', tbl.tableId):
@@ -480,6 +482,7 @@ class UserActions(object):
     for col, values in col_updates.iteritems():
       self.doModifyColumn(col.tableId, col.colId, values)
     self.doBulkUpdateFromPairs('_grist_Tables_column', col_updates.items())
+    make_acl_updates()
 
 
   @override_action('BulkUpdateRecord', '_grist_Tables_column')
@@ -531,6 +534,8 @@ class UserActions(object):
                    for key, value in values.iteritems()):
           raise ValueError("Cannot modify summary group-by column '%s'" % col.colId)
 
+    make_acl_updates = acl.prepare_acl_col_renames(self._docmodel, self, renames)
+
     for c, values in update_pairs:
       # Trigger ModifyColumn and RenameColumn as necessary
       schema_colinfo = select_keys(values, _modify_col_schema_props)
@@ -546,6 +551,7 @@ class UserActions(object):
                           widgetOptions='', displayCol=0)
 
     self.doBulkUpdateFromPairs(table_id, update_pairs)
+    make_acl_updates()
 
 
   @override_action('BulkUpdateRecord', '_grist_Views')