(core) Makes filter counts take other column filters into account

Summary:
Makes filter counts take other column filters into account.

 - Changes the summaries rows to reflect hidden rows:
    - hidden rows are added to the `Other Values` summary
    - show the unique number of other values as `Other Values (12)`
 - Also, adds a sort button to the column filter menu

Test Plan: Adds browser test.

Reviewers: paulfitz, jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D2861

app/client/models/ColumnFilter.ts

@@ -1,8 +1,8 @@
+import {ColumnFilterFunc, makeFilterFunc} from "app/common/ColumnFilterFunc";
 import {CellValue} from 'app/common/DocActions';
+import {FilterSpec, FilterState, makeFilterState} from "app/common/FilterState";
 import {nativeCompare} from 'app/common/gutil';
 import {Computed, Disposable, Observable} from 'grainjs';
-import {ColumnFilterFunc, makeFilterFunc} from "app/common/ColumnFilterFunc";
-import {FilterSpec, FilterState, makeFilterState} from "app/common/FilterState";
 
 /**
  * ColumnFilter implements a custom filter on a column, i.e. a filter that's diverged from what's

app/client/models/ColumnFilterMenuModel.ts

@@ -1,8 +1,8 @@
+import { ColumnFilter } from "app/client/models/ColumnFilter";
+import { localeCompare, nativeCompare } from "app/common/gutil";
+import { CellValue } from "app/plugin/GristData";
 import { Computed, Disposable, Observable } from "grainjs";
 import escapeRegExp = require("lodash/escapeRegExp");
-import { CellValue } from "app/plugin/GristData";
-import { localeCompare } from "app/common/gutil";
-import { ColumnFilter } from "./ColumnFilter";
 
 const MAXIMUM_SHOWN_FILTER_ITEMS = 500;
 
@@ -11,22 +11,37 @@ export interface IFilterCount {
   count: number;
 }
 
+type ICompare<T> = (a: T, b: T) => number
 
 export class ColumnFilterMenuModel extends Disposable {
 
   public readonly searchValue = Observable.create(this, '');
 
+  public readonly isSortedByCount = Observable.create(this, false);
+
   // computes a set of all keys that matches the search text.
   public readonly filterSet = Computed.create(this, this.searchValue, (_use, searchValue) => {
     const searchRegex = new RegExp(escapeRegExp(searchValue), 'i');
-    return new Set(this._valueCount.filter(([_, {label}]) => searchRegex.test(label)).map(([key]) => key));
+    return new Set(
+      this._valueCount
+        .filter(([_, {count}]) => count)
+        .filter(([_, {label}]) => searchRegex.test(label))
+        .map(([key]) => key)
+    );
   });
 
   // computes the sorted array of all values (ie: pair of key and IFilterCount) that matches the search text.
-  public readonly filteredValues = Computed.create(this, this.filterSet, (_use, filter) => {
-    return this._valueCount.filter(([key]) => filter.has(key))
-      .sort((a, b) => localeCompare(a[1].label, b[1].label));
-  });
+  public readonly filteredValues = Computed.create(
+    this, this.filterSet, this.isSortedByCount,
+    (_use, filter, isSortedByCount) => {
+      const comparator: ICompare<[CellValue, IFilterCount]> = isSortedByCount ?
+        (a, b) => nativeCompare(b[1].count, a[1].count) :
+        (a, b) => localeCompare(a[1].label, b[1].label);
+      return this._valueCount
+        .filter(([key]) => filter.has(key))
+        .sort(comparator);
+    }
+  );
 
   // computes the array of all values that does NOT matches the search text
   public readonly otherValues = Computed.create(this, this.filterSet, (_use, filter) => {
@@ -34,13 +49,15 @@ export class ColumnFilterMenuModel extends Disposable {
   });
 
   // computes the array of keys that matches the search text
-  public readonly filteredKeys = Computed.create(this, this.filteredValues, (_use, values) => (
-    values.map(([key]) => key)
-  ));
+  public readonly filteredKeys = Computed.create(this, this.filterSet, (_use, filter) => {
+    return this._valueCount
+      .filter(([key]) => filter.has(key))
+      .map(([key]) => key);
+  });
 
-  public readonly valuesBeyondLimit = Computed.create(this, this.filteredValues, (_use, values) => (
-    values.slice(this.limitShown)
-  ));
+  public readonly valuesBeyondLimit = Computed.create(this, this.filteredValues, (_use, filteredValues) => {
+    return filteredValues.slice(this.limitShown);
+  });
 
   constructor(public columnFilter: ColumnFilter, private _valueCount: Array<[CellValue, IFilterCount]>,
               public limitShown: number = MAXIMUM_SHOWN_FILTER_ITEMS) {

app/client/models/SectionFilter.ts

@@ -1,17 +1,21 @@
 import {KoArray} from 'app/client/lib/koArray';
+import {ColumnFilter} from 'app/client/models/ColumnFilter';
 import {ViewFieldRec} from 'app/client/models/DocModel';
 import {RowId} from 'app/client/models/rowset';
 import {TableData} from 'app/client/models/TableData';
-import {Computed, Disposable, MutableObsArray, obsArray, Observable} from 'grainjs';
-import {ColumnFilter} from './ColumnFilter';
-import {buildRowFilter, RowFilterFunc, RowValueFunc } from "app/common/RowFilterFunc";
-import {buildColFilter} from "app/common/ColumnFilterFunc";
+import {buildColFilter, ColumnFilterFunc} from 'app/common/ColumnFilterFunc';
+import {buildRowFilter, RowFilterFunc, RowValueFunc } from 'app/common/RowFilterFunc';
+import {Computed, Disposable, MutableObsArray, obsArray, Observable, UseCB} from 'grainjs';
+
+export {ColumnFilterFunc} from 'app/common/ColumnFilterFunc';
 
 interface OpenColumnFilter {
   fieldRef: number;
   colFilter: ColumnFilter;
 }
 
+type ColFilterCB = (field: ViewFieldRec, colFilter: ColumnFilterFunc|null) => ColumnFilterFunc|null;
+
 /**
  * SectionFilter represents a collection of column filters in place for a view section. It is created
  * out of `viewFields` and `tableData`, and provides a Computed `sectionFilterFunc` that users can
@@ -28,31 +32,22 @@ export class SectionFilter extends Disposable {
   private _openFilterOverride: Observable<OpenColumnFilter|null> = Observable.create(this, null);
   private _tempRows: MutableObsArray<RowId> = obsArray();
 
-  constructor(viewFields: ko.Computed<KoArray<ViewFieldRec>>, tableData: TableData) {
+  constructor(public viewFields: ko.Computed<KoArray<ViewFieldRec>>, private _tableData: TableData) {
     super();
 
     const columnFilterFunc = Computed.create(this, this._openFilterOverride, (use, openFilter) => {
-      const fields = use(use(viewFields).getObservable());
-      const funcs: Array<RowFilterFunc<RowId> | null> = fields.map(f => {
-        const filterFunc = (openFilter && openFilter.fieldRef === f.getRowId()) ?
-          use(openFilter.colFilter.filterFunc) :
-          buildColFilter(use(f.activeFilter), use(f.column).type());
-
-        const getter = tableData.getRowPropFunc(use(f.colId));
-
-        if (!filterFunc || !getter) { return null; }
-
-        return buildRowFilter(getter as RowValueFunc<RowId>, filterFunc);
-      })
-      .filter(f => f !== null); // Filter out columns that don't have a filter
-
-      return (rowId: RowId) => funcs.every(f => Boolean(f && f(rowId)));
+      const openFilterFilterFunc = openFilter && use(openFilter.colFilter.filterFunc);
+      function getFilterFunc(field: ViewFieldRec, colFilter: ColumnFilterFunc|null) {
+        if (openFilter?.fieldRef === field.getRowId()) {
+          return openFilterFilterFunc;
+        }
+        return colFilter;
+      }
+      return this._buildPlainFilterFunc(getFilterFunc, use);
     });
 
     this.sectionFilterFunc = Computed.create(this, columnFilterFunc, this._tempRows,
-      (_use, filterFunc, tempRows) => {
-      return (rowId: RowId) => tempRows.includes(rowId) || (typeof rowId !== 'number') || filterFunc(rowId);
-    });
+      (_use, filterFunc, tempRows) => this._addRowsToFilter(filterFunc, tempRows));
 
     // Prune temporary rowIds that are no longer being filtered out.
     this.autoDispose(columnFilterFunc.addListener(f => {
@@ -84,4 +79,39 @@ export class SectionFilter extends Disposable {
   public resetTemporaryRows() {
     this._tempRows.set([]);
   }
+
+  /**
+   * Builds a filter function that combines the filter function of all the fields. You can use
+   * `getFilterFunc(field, colFilter)` to customize the filter func for each field. It calls
+   * `getFilterFunc` right away. Also, all the rows that were added with `addTemporaryRow()` bypass
+   * the filter.
+   */
+  public buildFilterFunc(getFilterFunc: ColFilterCB, use: UseCB) {
+    return this._addRowsToFilter(this._buildPlainFilterFunc(getFilterFunc, use), this._tempRows.get());
+  }
+
+  private _addRowsToFilter(filterFunc: RowFilterFunc<RowId>, rows: RowId[]) {
+    return (rowId: RowId) => rows.includes(rowId) || (typeof rowId !== 'number') || filterFunc(rowId);
+  }
+
+  /**
+   * Internal that helps build a filter function that combines the filter function of all
+   * fields. You can use `getFilterFunc(field, colFilter)` to customize the filter func for each
+   * field. It calls `getFilterFunc` right away
+   */
+  private _buildPlainFilterFunc(getFilterFunc: ColFilterCB, use: UseCB): RowFilterFunc<RowId> {
+    const fields = use(use(this.viewFields).getObservable());
+    const funcs: Array<RowFilterFunc<RowId> | null> = fields.map(f => {
+      const colFilter = buildColFilter(use(f.activeFilter), use(use(f.column).type));
+      const filterFunc = getFilterFunc(f, colFilter);
+
+      const getter = this._tableData.getRowPropFunc(f.colId.peek());
+
+      if (!filterFunc || !getter) { return null; }
+
+      return buildRowFilter(getter as RowValueFunc<RowId>, filterFunc);
+    }).filter(f => f !== null); // Filter out columns that don't have a filter
+
+    return (rowId: RowId) => funcs.every(f => Boolean(f && f(rowId)));
+  }
 }