(core) Allow filtering hidden columns

Summary:
Existing filters are now moved out of fields
and into a new metadata table for filters, and the
client is updated to retrieve/update/save filters from
the new table. This enables storing of filters for
columns that don't have fields (notably, hidden columns).

Test Plan: Browser and server tests.

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D3138

app/client/ui/ColumnFilterMenu.ts

@@ -6,7 +6,8 @@
 
 import {allInclusive, ColumnFilter} from 'app/client/models/ColumnFilter';
 import {ColumnFilterMenuModel, IFilterCount} from 'app/client/models/ColumnFilterMenuModel';
-import {ViewFieldRec, ViewSectionRec} from 'app/client/models/DocModel';
+import {ColumnRec, ViewFieldRec, ViewSectionRec} from 'app/client/models/DocModel';
+import {FilterInfo} from 'app/client/models/entities/ViewSectionRec';
 import {RowId, RowSource} from 'app/client/models/rowset';
 import {ColumnFilterFunc, SectionFilter} from 'app/client/models/SectionFilter';
 import {TableData} from 'app/client/models/TableData';
@@ -278,21 +279,22 @@ function formatUniqueCount(values: Array<[CellValue, IFilterCount]>) {
 /**
  * Returns content for the newly created columnFilterMenu; for use with setPopupToCreateDom().
  */
-export function createFilterMenu(openCtl: IOpenController, sectionFilter: SectionFilter, field: ViewFieldRec,
+export function createFilterMenu(openCtl: IOpenController, sectionFilter: SectionFilter, filterInfo: FilterInfo,
                                  rowSource: RowSource, tableData: TableData, onClose: () => void = noop) {
   // Go through all of our shown and hidden rows, and count them up by the values in this column.
-  const columnType = field.column().type.peek();
-  const {keyMapFunc, labelMapFunc} = getMapFuncs(columnType, tableData, field);
-  const activeFilterBar = field.viewSection.peek().activeFilterBar;
+  const fieldOrColumn = filterInfo.fieldOrColumn;
+  const columnType = fieldOrColumn.origCol.peek().type.peek();
+  const {keyMapFunc, labelMapFunc} = getMapFuncs(columnType, tableData, filterInfo.fieldOrColumn);
+  const activeFilterBar = sectionFilter.viewSection.activeFilterBar;
 
-  function getFilterFunc(f: ViewFieldRec, colFilter: ColumnFilterFunc|null) {
-    return f.getRowId() === field.getRowId() ? null : colFilter;
+  function getFilterFunc(col: ViewFieldRec|ColumnRec, colFilter: ColumnFilterFunc|null) {
+    return col.getRowId() === fieldOrColumn.getRowId() ? null : colFilter;
   }
   const filterFunc = Computed.create(null, use => sectionFilter.buildFilterFunc(getFilterFunc, use));
   openCtl.autoDispose(filterFunc);
 
-  const columnFilter = ColumnFilter.create(openCtl, field.activeFilter.peek(), columnType);
-  sectionFilter.setFilterOverride(field.getRowId(), columnFilter); // Will be removed on menu disposal
+  const columnFilter = ColumnFilter.create(openCtl, filterInfo.filter.peek(), columnType);
+  sectionFilter.setFilterOverride(fieldOrColumn.getRowId(), columnFilter); // Will be removed on menu disposal
 
   const [allRows, hiddenRows] = partition(Array.from(rowSource.getAllRows()), filterFunc.get());
   const valueCounts: Map<CellValue, {label: string, count: number}> = new Map();
@@ -310,12 +312,15 @@ export function createFilterMenu(openCtl: IOpenController, sectionFilter: Sectio
     doSave: (reset: boolean = false) => {
       const spec = columnFilter.makeFilterJson();
       // If filter is moot and filter bar is hidden, let's remove the filter.
-      field.activeFilter((spec === allInclusive && !activeFilterBar.peek()) ? '' : spec);
+      sectionFilter.viewSection.setFilter(
+        fieldOrColumn.origCol().origColRef(),
+        spec === allInclusive && !activeFilterBar.peek() ? '' : spec
+      );
       if (reset) {
         sectionFilter.resetTemporaryRows();
       }
     },
-    renderValue: getRenderFunc(columnType, field),
+    renderValue: getRenderFunc(columnType, fieldOrColumn),
   });
 }
 
@@ -330,10 +335,10 @@ export function createFilterMenu(openCtl: IOpenController, sectionFilter: Sectio
  * Used by ColumnFilterMenu to compute counts of unique cell
  * values and display them with an appropriate label.
  */
-function getMapFuncs(columnType: string, tableData: TableData, field: ViewFieldRec) {
-  const keyMapFunc = tableData.getRowPropFunc(field.column().colId())!;
-  const labelGetter = tableData.getRowPropFunc(field.displayColModel().colId())!;
-  const formatter = field.createVisibleColFormatter();
+function getMapFuncs(columnType: string, tableData: TableData, fieldOrColumn: ViewFieldRec|ColumnRec) {
+  const keyMapFunc = tableData.getRowPropFunc(fieldOrColumn.colId())!;
+  const labelGetter = tableData.getRowPropFunc(fieldOrColumn.displayColModel().colId())!;
+  const formatter = fieldOrColumn.createVisibleColFormatter();
 
   let labelMapFunc: (rowId: number) => string | string[];
   if (isRefListType(columnType)) {
@@ -357,9 +362,9 @@ function getMapFuncs(columnType: string, tableData: TableData, field: ViewFieldR
  * column types by rendering their values as colored tokens instead of
  * text.
  */
-function getRenderFunc(columnType: string, field: ViewFieldRec) {
+function getRenderFunc(columnType: string, fieldOrColumn: ViewFieldRec|ColumnRec) {
   if (['Choice', 'ChoiceList'].includes(columnType)) {
-    const options = field.column().widgetOptionsJson.peek();
+    const options = fieldOrColumn.widgetOptionsJson.peek();
     const choiceSet: Set<string> = new Set(options.choices || []);
     const choiceOptions: ChoiceOptions = options.choiceOptions || {};
 
@@ -460,13 +465,14 @@ interface IColumnFilterMenuOptions extends IPopupOptions {
 }
 
 // Helper to attach the column filter menu.
-export function attachColumnFilterMenu(viewSection: ViewSectionRec, field: ViewFieldRec,
+export function attachColumnFilterMenu(viewSection: ViewSectionRec, filterInfo: FilterInfo,
                                        popupOptions: IColumnFilterMenuOptions): DomElementMethod {
   const options = {...defaultPopupOptions, ...popupOptions};
   return (elem) => {
     const instance = viewSection.viewInstance();
     if (instance && instance.createFilterMenu) { // Should be set if using BaseView
-      setPopupToCreateDom(elem, ctl => instance.createFilterMenu(ctl, field, popupOptions.onCloseContent), options);
+      setPopupToCreateDom(elem, ctl =>
+        instance.createFilterMenu(ctl, filterInfo, popupOptions.onCloseContent), options);
     }
   };
 }

app/client/ui/FilterBar.ts

@@ -1,5 +1,6 @@
 import { allInclusive } from "app/client/models/ColumnFilter";
-import { ViewFieldRec, ViewSectionRec } from "app/client/models/DocModel";
+import { ColumnRec, ViewFieldRec, ViewSectionRec } from "app/client/models/DocModel";
+import { FilterInfo } from "app/client/models/entities/ViewSectionRec";
 import { attachColumnFilterMenu } from "app/client/ui/ColumnFilterMenu";
 import { cssButton, cssButtonGroup } from "app/client/ui2018/buttons";
 import { colors, testId } from "app/client/ui2018/cssVars";
@@ -9,46 +10,46 @@ import { dom, IDisposableOwner, IDomArgs, styled } from "grainjs";
 import { IMenuOptions, PopupControl } from "popweasel";
 
 export function filterBar(_owner: IDisposableOwner, viewSection: ViewSectionRec) {
-  const popupControls = new WeakMap<ViewFieldRec, PopupControl>();
+  const popupControls = new WeakMap<ColumnRec, PopupControl>();
   return cssFilterBar(
     testId('filter-bar'),
-    dom.forEach(viewSection.filteredFields, (field) => makeFilterField(viewSection, field, popupControls)),
+    dom.forEach(viewSection.activeFilters, (filterInfo) => makeFilterField(viewSection, filterInfo, popupControls)),
     makePlusButton(viewSection, popupControls),
   );
 }
 
-function makeFilterField(viewSection: ViewSectionRec, field: ViewFieldRec,
-                         popupControls: WeakMap<ViewFieldRec, PopupControl>) {
+function makeFilterField(viewSection: ViewSectionRec, filterInfo: FilterInfo,
+                         popupControls: WeakMap<ColumnRec, PopupControl>) {
   return cssFilterBarItem(
     testId('filter-field'),
     primaryButton(
       testId('btn'),
       cssIcon('FilterSimple'),
-      cssMenuTextLabel(dom.text(field.label)),
-      cssBtn.cls('-grayed', field.activeFilter.isSaved),
-      attachColumnFilterMenu(viewSection, field, {
+      cssMenuTextLabel(dom.text(filterInfo.fieldOrColumn.label)),
+      cssBtn.cls('-grayed', filterInfo.filter.isSaved),
+      attachColumnFilterMenu(viewSection, filterInfo, {
         placement: 'bottom-start', attach: 'body',
-        trigger: ['click', (_el, popupControl) => popupControls.set(field, popupControl)]
+        trigger: ['click', (_el, popupControl) => popupControls.set(filterInfo.fieldOrColumn.origCol(), popupControl)]
       }),
     ),
     deleteButton(
       testId('delete'),
       cssIcon('CrossSmall'),
-      cssBtn.cls('-grayed', field.activeFilter.isSaved),
-      dom.on('click', () => field.activeFilter('')),
+      cssBtn.cls('-grayed', filterInfo.filter.isSaved),
+      dom.on('click', () => viewSection.setFilter(filterInfo.fieldOrColumn.origCol().origColRef(), '')),
     )
   );
 }
 
-export function addFilterMenu(fields: ViewFieldRec[], popupControls: WeakMap<ViewFieldRec, PopupControl>,
-                              options?: IMenuOptions) {
+export function addFilterMenu(filters: FilterInfo[], viewSection: ViewSectionRec,
+                              popupControls: WeakMap<ColumnRec, PopupControl>, options?: IMenuOptions) {
   return (
     menu((ctl) => [
-      ...fields.map((f) => (
+      ...filters.map((filterInfo) => (
         menuItemAsync(
-          () => turnOnAndOpenFilter(f, popupControls),
-          f.label.peek(),
-          dom.cls('disabled', f.isFiltered),
+          () => turnOnAndOpenFilter(filterInfo.fieldOrColumn, viewSection, popupControls),
+          filterInfo.fieldOrColumn.label.peek(),
+          dom.cls('disabled', filterInfo.isFiltered),
           testId('add-filter-item'),
         )
       )),
@@ -62,19 +63,20 @@ export function addFilterMenu(fields: ViewFieldRec[], popupControls: WeakMap<Vie
   );
 }
 
-function turnOnAndOpenFilter(f: ViewFieldRec, popupControls: WeakMap<ViewFieldRec, PopupControl>) {
-   f.activeFilter(allInclusive);
-   popupControls.get(f)?.open();
+function turnOnAndOpenFilter(fieldOrColumn: ViewFieldRec|ColumnRec, viewSection: ViewSectionRec,
+                             popupControls: WeakMap<ColumnRec, PopupControl>) {
+  viewSection.setFilter(fieldOrColumn.origCol().origColRef(), allInclusive);
+  popupControls.get(fieldOrColumn.origCol())?.open();
 }
 
-function makePlusButton(viewSectionRec: ViewSectionRec, popupControls: WeakMap<ViewFieldRec, PopupControl>) {
+function makePlusButton(viewSectionRec: ViewSectionRec, popupControls: WeakMap<ColumnRec, PopupControl>) {
   return dom.domComputed((use) => {
-    const fields = use(use(viewSectionRec.viewFields).getObservable());
-    const anyFilter = fields.find((f) => use(f.isFiltered));
+    const filters = use(viewSectionRec.filters);
+    const anyFilter = use(viewSectionRec.activeFilters).length > 0;
     return cssPlusButton(
       cssBtn.cls('-grayed'),
       cssIcon('Plus'),
-      addFilterMenu(fields, popupControls),
+      addFilterMenu(filters, viewSectionRec, popupControls),
       anyFilter ? null : cssPlusLabel('Add Filter'),
       testId('add-filter-btn')
     );

app/client/ui/ViewSectionMenu.ts

@@ -1,5 +1,6 @@
 import { reportError } from 'app/client/models/AppModel';
-import { ColumnRec, DocModel, ViewFieldRec, ViewRec, ViewSectionRec } from 'app/client/models/DocModel';
+import { ColumnRec, DocModel, ViewRec, ViewSectionRec } from 'app/client/models/DocModel';
+import { FilterInfo } from 'app/client/models/entities/ViewSectionRec';
 import { CustomComputed } from 'app/client/models/modelUtil';
 import { attachColumnFilterMenu } from 'app/client/ui/ColumnFilterMenu';
 import { addFilterMenu } from 'app/client/ui/FilterBar';
@@ -35,8 +36,8 @@ function doRevert(viewSection: ViewSectionRec) {
 export function viewSectionMenu(owner: IDisposableOwner, docModel: DocModel, viewSection: ViewSectionRec,
                                 viewModel: ViewRec, isReadonly: Observable<boolean>) {
 
-  const popupControls = new WeakMap<ViewFieldRec, PopupControl>();
-  const anyFilter = Computed.create(owner, (use) => Boolean(use(viewSection.filteredFields).length));
+  const popupControls = new WeakMap<ColumnRec, PopupControl>();
+  const anyFilter = Computed.create(owner, (use) => Boolean(use(viewSection.activeFilters).length));
 
   const displaySaveObs: Computed<boolean> = Computed.create(owner, (use) => (
     use(viewSection.filterSpecChanged)
@@ -65,8 +66,8 @@ export function viewSectionMenu(owner: IDisposableOwner, docModel: DocModel, vie
             return makeSortPanel(viewSection, use(viewSection.activeSortSpec),
                                  (row: number) => docModel.columns.getRowModel(row));
           }),
-          dom.domComputed(viewSection.filteredFields, fields =>
-                          makeFilterPanel(viewSection, fields, popupControls, () => ctl.close())),
+          dom.domComputed(viewSection.activeFilters, filters =>
+                          makeFilterPanel(viewSection, filters, popupControls, () => ctl.close())),
           makeAddFilterButton(viewSection, popupControls),
           makeFilterBarToggle(viewSection.activeFilterBar),
           dom.domComputed(displaySaveObs, displaySave => [
@@ -139,14 +140,13 @@ function makeSortPanel(section: ViewSectionRec, sortSpec: Sort.SortSpec, getColu
   ];
 }
 
-export function makeAddFilterButton(viewSectionRec: ViewSectionRec,
-                                    popupControls: WeakMap<ViewFieldRec, PopupControl>) {
+export function makeAddFilterButton(viewSectionRec: ViewSectionRec, popupControls: WeakMap<ColumnRec, PopupControl>) {
   return dom.domComputed((use) => {
-    const fields = use(use(viewSectionRec.viewFields).getObservable());
+    const filters = use(viewSectionRec.filters);
     return cssMenuText(
       cssMenuIconWrapper(
         cssIcon('Plus'),
-        addFilterMenu(fields, popupControls, {
+        addFilterMenu(filters, viewSectionRec, popupControls, {
           placement: 'bottom-end',
           // Attach content to triggerElem's parent, which is needed to prevent view section menu to
           // close when clicking an item of the add filter menu.
@@ -179,32 +179,37 @@ export function makeFilterBarToggle(activeFilterBar: CustomComputed<boolean>) {
 }
 
 
-function makeFilterPanel(section: ViewSectionRec, filteredFields: ViewFieldRec[],
-                         popupControls: WeakMap<ViewFieldRec, PopupControl>,
+function makeFilterPanel(section: ViewSectionRec, activeFilters: FilterInfo[],
+                         popupControls: WeakMap<ColumnRec, PopupControl>,
                          onCloseContent: () => void) {
-  const fields = filteredFields.map(field => {
-    const fieldChanged = Computed.create(null, fromKo(field.activeFilter.isSaved), (_use, isSaved) => !isSaved);
+  const filters = activeFilters.map(filterInfo => {
+    const filterChanged = Computed.create(null, fromKo(filterInfo.filter.isSaved), (_use, isSaved) => !isSaved);
     return cssMenuText(
-      dom.autoDispose(fieldChanged),
       cssMenuIconWrapper(
-        cssMenuIconWrapper.cls('-changed', fieldChanged),
+        cssMenuIconWrapper.cls('-changed', filterChanged),
         cssIcon('FilterSimple'),
-        attachColumnFilterMenu(section, field, {
+        attachColumnFilterMenu(section, filterInfo, {
           placement: 'bottom-end',
-          trigger: ['click', (_el, popupControl) => popupControls.set(field, popupControl)],
+          trigger: [
+            'click',
+            (_el, popupControl) => popupControls.set(filterInfo.fieldOrColumn.origCol(), popupControl)
+          ],
           onCloseContent,
         }),
         testId('filter-icon'),
       ),
-      cssMenuTextLabel(field.label()),
-      cssMenuIconWrapper(cssIcon('Remove', testId('btn-remove-filter')), dom.on('click', () => field.activeFilter(''))),
+      cssMenuTextLabel(filterInfo.fieldOrColumn.label()),
+      cssMenuIconWrapper(cssIcon('Remove',
+        dom.on('click', () => section.setFilter(filterInfo.fieldOrColumn.origCol().origColRef(), ''))),
+        testId('btn-remove-filter')
+      ),
       testId('filter-col')
     );
   });
 
   return [
     cssMenuInfoHeader('Filtered by', {style: 'margin-top: 4px'}, testId('heading-filtered')),
-    filteredFields.length > 0 ? fields : cssGrayedMenuText('(Not filtered)')
+    activeFilters.length > 0 ? filters : cssGrayedMenuText('(Not filtered)')
   ];
 }