(core) Add a row to summary tables grouped by list column(s) corresponding to empty lists

Summary:
Adds some special handling to summary table and lookup logic:

- Source rows with empty choicelists/reflists get a corresponding summary row with an empty string/reference when grouping by that column, instead of excluding them from any group
- Adds a new `QueryOperation` 'empty' in the client which is used in `LinkingState`, `QuerySet`, and `recursiveMoveToCursorPos` to match empty lists in source tables against falsy values in linked summary tables.
- Adds a new parameter `match_empty` to the Python `CONTAINS` function so that regular formulas can implement the same behaviour as summary tables. See https://grist.slack.com/archives/C0234CPPXPA/p1654030490932119
- Uses the new `match_empty` argument in the formula generated for the `group` column when detaching a summary table.

Test Plan: Updated and extended Python and nbrowser tests of summary tables grouped by choicelists to test for new behaviour with empty lists.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3471

app/client/components/GristDoc.ts

@@ -55,7 +55,7 @@ import {CommDocUsage, CommDocUserAction} from 'app/common/CommTypes';
 import {DisposableWithEvents} from 'app/common/DisposableWithEvents';
 import {isSchemaAction, UserAction} from 'app/common/DocActions';
 import {OpenLocalDocResult} from 'app/common/DocListAPI';
-import {isList, isRefListType, RecalcWhen} from 'app/common/gristTypes';
+import {isList, isListType, isRefListType, RecalcWhen} from 'app/common/gristTypes';
 import {HashLink, IDocPage, isViewDocPage, SpecialDocPage, ViewDocPage} from 'app/common/gristUrls';
 import {undef, waitObs} from 'app/common/gutil';
 import {LocalPlugin} from "app/common/plugin";
@@ -856,9 +856,12 @@ export class GristDoc extends DisposableWithEvents {
             // must be a summary -- otherwise dealt with earlier.
             const destTable = await this._getTableData(section);
             for (const srcCol of srcSection.table.peek().groupByColumns.peek()) {
-              const filterColId = srcCol.summarySource.peek().colId.peek();
+              const filterCol = srcCol.summarySource.peek();
+              const filterColId = filterCol.colId.peek();
               controller = destTable.getValue(cursorPos.rowId, filterColId);
-              query.operations[filterColId] = 'in';
+              // If the source groupby column is a ChoiceList or RefList, then null or '' in the summary table
+              // should match against an empty list in the source table.
+              query.operations[filterColId] = isListType(filterCol.type.peek()) && !controller ? 'empty' : 'in';
               query.filters[filterColId] = isList(controller) ? controller.slice(1) : [controller];
             }
           }

app/client/components/LinkingState.ts

@@ -7,14 +7,14 @@ import {ViewSectionRec} from "app/client/models/entities/ViewSectionRec";
 import {RowId} from "app/client/models/rowset";
 import {LinkConfig} from "app/client/ui/selectBy";
 import {ClientQuery, QueryOperation} from "app/common/ActiveDocAPI";
-import {isList, isRefListType} from "app/common/gristTypes";
+import {isList, isListType, isRefListType} from "app/common/gristTypes";
 import * as gutil from "app/common/gutil";
 import {encodeObject} from 'app/plugin/objtypes';
 import {Disposable, toKo} from "grainjs";
 import * as  ko from "knockout";
+import identity = require('lodash/identity');
 import mapValues = require('lodash/mapValues');
 import pickBy = require('lodash/pickBy');
-import identity = require('lodash/identity');
 
 
 /**
@@ -124,18 +124,14 @@ export class LinkingState extends Disposable {
           const srcValue = srcTableData.getValue(srcRowId as number, colId);
           result.filters[colId] = [srcValue];
           result.operations[colId] = 'in';
-          if (isDirectSummary) {
-            const tgtColType = col.type();
-            if (tgtColType === 'ChoiceList' || tgtColType.startsWith('RefList:')) {
-              result.operations[colId] = 'intersects';
-            }
+          if (isDirectSummary && isListType(col.type())) {
+            // If the source groupby column is a ChoiceList or RefList, then null or '' in the summary table
+            // should match against an empty list in the source table.
+            result.operations[colId] = srcValue ? 'intersects' : 'empty';
           }
         }
         _filterColValues(result);
       }
-    } else if (isSummaryOf(tgtSection.table(), srcSection.table())) {
-      // TODO: We should move the cursor, but don't currently it for summaries. For that, we need a
-      // column or map representing the inverse of summary table's "group" column.
     } else if (srcSection.parentKey() === 'custom') {
       this.filterColValues = this._srcCustomFilter('id', 'in');
     } else {

app/client/models/QuerySet.ts

@@ -33,16 +33,16 @@ import {TableData} from 'app/client/models/TableData';
 import {ActiveDocAPI, ClientQuery, QueryOperation} from 'app/common/ActiveDocAPI';
 import {CellValue, TableDataAction} from 'app/common/DocActions';
 import {DocData} from 'app/common/DocData';
+import {isList} from "app/common/gristTypes";
 import {nativeCompare} from 'app/common/gutil';
 import {IRefCountSub, RefCountMap} from 'app/common/RefCountMap';
-import {TableData as BaseTableData} from 'app/common/TableData';
 import {RowFilterFunc} from 'app/common/RowFilterFunc';
+import {TableData as BaseTableData} from 'app/common/TableData';
 import {tbind} from 'app/common/tbind';
+import {decodeObject} from "app/plugin/objtypes";
 import {Disposable, Holder, IDisposableOwnerT} from 'grainjs';
 import * as ko from 'knockout';
 import debounce = require('lodash/debounce');
-import {isList} from "app/common/gristTypes";
-import {decodeObject} from "app/plugin/objtypes";
 
 // Limit on the how many rows to request for OnDemand tables.
 const ON_DEMAND_ROW_LIMIT = 10000;
@@ -310,17 +310,21 @@ export function getFilterFunc(docData: DocData, query: ClientQuery): RowFilterFu
     (colId) => {
       const getter = tableData.getRowPropFunc(colId)!;
       const values = new Set(query.filters[colId]);
-      switch (query.operations![colId]) {
+      switch (query.operations[colId]) {
         case "intersects":
           return (rowId: RowId) => {
             const value = getter(rowId) as CellValue;
             return isList(value) &&
               (decodeObject(value) as unknown[]).some(v => values.has(v));
           };
+        case "empty":
+          return (rowId: RowId) => {
+            const value = getter(rowId);
+            // `isList(value) && value.length === 1` means `value == ['L']` i.e. an empty list
+            return !value || isList(value) && value.length === 1;
+          };
         case "in":
           return (rowId: RowId) => values.has(getter(rowId));
-        default:
-          throw new Error("Unknown operation");
       }
     });
   return (rowId: RowId) => colFuncs.every(f => f(rowId));
@@ -342,7 +346,7 @@ function convertQueryToRefs(docModel: DocModel, query: ClientQuery): QueryRefs {
     const values = query.filters[colId];
     // Keep filter values sorted by value, for consistency.
     values.sort(nativeCompare);
-    return [colRefsByColId[colId], query.operations![colId], values] as FilterTuple;
+    return [colRefsByColId[colId], query.operations[colId], values] as FilterTuple;
   });
   // Keep filters sorted by colRef, for consistency.
   filterTuples.sort((a, b) =>

app/common/ActiveDocAPI.ts

@@ -121,7 +121,10 @@ export interface QueryFilters {
   [colId: string]: any[];
 }
 
-export type QueryOperation = "in" | "intersects";
+// - in: value should be contained in filters array
+// - intersects: value should be a list with some overlap with filters array
+// - empty: value should be falsy (e.g. null) or an empty list, filters is ignored
+export type QueryOperation = "in" | "intersects" | "empty";
 
 /**
  * Response from useQuerySet(). A query returns data AND creates a subscription to receive