(core) Allow filtering hidden columns

Summary:
Existing filters are now moved out of fields
and into a new metadata table for filters, and the
client is updated to retrieve/update/save filters from
the new table. This enables storing of filters for
columns that don't have fields (notably, hidden columns).

Test Plan: Browser and server tests.

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D3138

sandbox/grist/docmodel.py

@@ -107,6 +107,11 @@ class MetaTableExtras(object):
   class _grist_Views_section(object):
     fields = _record_set('_grist_Views_section_field', 'parentId', sort_by='parentPos')
 
+  class _grist_Filters(object):
+    def setAutoRemove(rec, table):
+      """Marks the filter for removal if its column no longer exists."""
+      table.docmodel.setAutoRemove(rec, not rec.colRef)
+
 
 def enhance_model(model_class):
   """
@@ -159,6 +164,7 @@ class DocModel(object):
     self.pages                   = self._prep_table("_grist_Pages")
     self.aclResources            = self._prep_table("_grist_ACLResources")
     self.aclRules                = self._prep_table("_grist_ACLRules")
+    self.filters                 = self._prep_table("_grist_Filters")
 
   def _prep_table(self, name):
     """

sandbox/grist/migrations.py

@@ -807,3 +807,36 @@ def migration24(tdset):
       schema.make_column("actions", "Text"),  # JSON
     ]),
   ])
+
+@migration(schema_version=25)
+def migration25(tdset):
+  """
+  Add _grist_Filters table and populate based on existing filters stored
+  in _grist_Views_section_field.
+
+  From this version on, filter in _grist_Views_section_field is deprecated.
+  """
+  doc_actions = [
+    actions.AddTable('_grist_Filters', [
+      schema.make_column("viewSectionRef", "Ref:_grist_Views_section"),
+      schema.make_column("colRef", "Ref:_grist_Tables_column"),
+      schema.make_column("filter", "Text"),
+    ])
+  ]
+
+  # Move existing field filters to _grist_Filters.
+  fields = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Views_section_field']))
+  col_info = { 'filter': [], 'colRef': [], 'viewSectionRef': [] }
+  for f in fields:
+    if not f.filter:
+      continue
+
+    col_info['filter'].append(f.filter)
+    col_info['colRef'].append(f.colRef)
+    col_info['viewSectionRef'].append(f.parentId)
+
+  num_filters = len(col_info['filter'])
+  if num_filters > 0:
+    doc_actions.append(actions.BulkAddRecord('_grist_Filters', [None] * num_filters, col_info))
+
+  return tdset.apply_doc_actions(doc_actions)

sandbox/grist/schema.py

@@ -15,7 +15,7 @@ import six
 
 import actions
 
-SCHEMA_VERSION = 24
+SCHEMA_VERSION = 25
 
 def make_column(col_id, col_type, formula='', isFormula=False):
   return {
@@ -202,11 +202,8 @@ def schema_create_actions():
       make_column("displayCol",   "Ref:_grist_Tables_column"),
       # For Ref cols only, may override the column to be displayed fromin the pointed-to table.
       make_column("visibleCol",   "Ref:_grist_Tables_column"),
-      # JSON string describing the default filter as map from either an `included` or an
-      # `excluded` string to an array of column values:
-      # Ex1: { included: ['foo', 'bar'] }
-      # Ex2: { excluded: ['apple', 'orange'] }
-      make_column("filter",       "Text")
+      # DEPRECATED: replaced with _grist_Filters in version 25. Do not remove or reuse.
+      make_column("filter",       "Text"),
     ]),
 
     # The code for all of the validation rules available to a Grist document
@@ -301,6 +298,16 @@ def schema_create_actions():
       make_column('parent', 'Ref:_grist_ACLPrincipals'),
       make_column('child',  'Ref:_grist_ACLPrincipals'),
     ]),
+
+    actions.AddTable('_grist_Filters', [
+      make_column("viewSectionRef", "Ref:_grist_Views_section"),
+      make_column("colRef",         "Ref:_grist_Tables_column"),
+      # JSON string describing the default filter as map from either an `included` or an
+      # `excluded` string to an array of column values:
+      # Ex1: { included: ['foo', 'bar'] }
+      # Ex2: { excluded: ['apple', 'orange'] }
+      make_column("filter",         "Text")
+    ]),
   ]