(core) Filter rows based on linked widgets when exporting view

Summary:
Fixes a problem reported here: https://community.getgrist.com/t/exporting-the-records-in-a-linked-view/2556/4

The download CSV/Excel link now contains an additional `linkingFilter` URL parameter containing JSON-encoded `filters` and `operations`. This object is originally created in the frontend in `LinkingState`, and previously it was only used internally in the frontend. It would make its way via `QuerySetManager` to `QuerySet.getFilterFunc` where the actual filtering logic happened. Now most of that logic has been moved to a similar function in `common`. The new function works with a new interface `ColumnGettersByColId` which abstract over the different ways data is accessed in the client and server in this context. There's no significant new logic in the diff, just refactoring and wiring.

Test Plan: Expanded two `nbrowser/SelectBy*.ts` test suites to also check the contents of a downloaded CSV in different linking scenarios.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3961

app/client/components/GristDoc.ts

@@ -57,7 +57,7 @@ import {invokePrompt} from 'app/client/ui2018/modals';
 import {DiscussionPanel} from 'app/client/widgets/DiscussionEditor';
 import {FieldEditor} from "app/client/widgets/FieldEditor";
 import {MinimalActionGroup} from 'app/common/ActionGroup';
-import {ClientQuery} from "app/common/ActiveDocAPI";
+import {ClientQuery, FilterColValues} from "app/common/ActiveDocAPI";
 import {CommDocChatter, CommDocUsage, CommDocUserAction} from 'app/common/CommTypes';
 import {delay} from 'app/common/delay';
 import {DisposableWithEvents} from 'app/common/DisposableWithEvents';
@@ -1518,18 +1518,20 @@ export class GristDoc extends DisposableWithEvents {
   }
 
   private _getDocApiDownloadParams() {
-    const filters = this.viewModel.activeSection.peek().activeFilters.get().map(filterInfo => ({
+    const activeSection = this.viewModel.activeSection();
+    const filters = activeSection.activeFilters.get().map(filterInfo => ({
       colRef: filterInfo.fieldOrColumn.origCol().origColRef(),
       filter: filterInfo.filter()
     }));
+    const linkingFilter: FilterColValues = activeSection.linkingFilter();
 
-    const params = {
+    return {
       viewSection: this.viewModel.activeSectionId(),
-      tableId: this.viewModel.activeSection().table().tableId(),
-      activeSortSpec: JSON.stringify(this.viewModel.activeSection().activeSortSpec()),
+      tableId: activeSection.table().tableId(),
+      activeSortSpec: JSON.stringify(activeSection.activeSortSpec()),
       filters: JSON.stringify(filters),
+      linkingFilter: JSON.stringify(linkingFilter),
     };
-    return params;
   }
 
   /**

app/client/components/LinkingState.ts

@@ -6,7 +6,7 @@ import {TableRec} from "app/client/models/entities/TableRec";
 import {ViewSectionRec} from "app/client/models/entities/ViewSectionRec";
 import {UIRowId} from "app/common/TableData";
 import {LinkConfig} from "app/client/ui/selectBy";
-import {ClientQuery, QueryOperation} from "app/common/ActiveDocAPI";
+import {FilterColValues, QueryOperation} from "app/common/ActiveDocAPI";
 import {isList, isListType, isRefListType} from "app/common/gristTypes";
 import * as gutil from "app/common/gutil";
 import {encodeObject} from 'app/plugin/objtypes';
@@ -34,8 +34,6 @@ function isSummaryOf(summary: TableRec, detail: TableRec): boolean {
     gutil.isSubset(summary.summarySourceColRefs(), detail.summarySourceColRefs()));
 }
 
-export type FilterColValues = Pick<ClientQuery, "filters" | "operations">;
-
 /**
  * Maintains state useful for linking sections, i.e. auto-filtering and auto-scrolling.
  * Exposes .filterColValues, which is either null or a computed evaluating to a filtering object;

app/client/models/ClientColumnGetters.ts

@@ -1,8 +1,9 @@
 import DataTableModel from 'app/client/models/DataTableModel';
-import { ColumnGetter, ColumnGetters } from 'app/common/ColumnGetters';
+import {ColumnGetter, ColumnGetters, ColumnGettersByColId} from 'app/common/ColumnGetters';
 import * as gristTypes from 'app/common/gristTypes';
-import { choiceGetter } from 'app/common/SortFunc';
-import { Sort } from 'app/common/SortSpec';
+import {choiceGetter} from 'app/common/SortFunc';
+import {Sort} from 'app/common/SortSpec';
+import {TableData} from 'app/common/TableData';
 
 /**
  *
@@ -56,3 +57,13 @@ export class ClientColumnGetters implements ColumnGetters {
     return this.getColGetter(manualSortCol.getRowId());
   }
 }
+
+
+export class ClientColumnGettersByColId implements ColumnGettersByColId {
+  constructor(private _tableData: TableData) {
+  }
+
+  public getColGetterByColId(colId: string): ColumnGetter {
+    return this._tableData.getRowPropFunc(colId);
+  }
+}

app/client/models/QuerySet.ts

@@ -31,17 +31,16 @@ import {DocModel} from 'app/client/models/DocModel';
 import {BaseFilteredRowSource, RowList, RowSource} from 'app/client/models/rowset';
 import {TableData} from 'app/client/models/TableData';
 import {ActiveDocAPI, ClientQuery, QueryOperation} from 'app/common/ActiveDocAPI';
-import {CellValue, TableDataAction} from 'app/common/DocActions';
+import {TableDataAction} from 'app/common/DocActions';
 import {DocData} from 'app/common/DocData';
-import {isList} from "app/common/gristTypes";
 import {nativeCompare} from 'app/common/gutil';
 import {IRefCountSub, RefCountMap} from 'app/common/RefCountMap';
-import {RowFilterFunc} from 'app/common/RowFilterFunc';
+import {getLinkingFilterFunc, RowFilterFunc} from 'app/common/RowFilterFunc';
 import {TableData as BaseTableData, UIRowId} from 'app/common/TableData';
 import {tbind} from 'app/common/tbind';
-import {decodeObject} from "app/plugin/objtypes";
 import {Disposable, Holder, IDisposableOwnerT} from 'grainjs';
 import * as ko from 'knockout';
+import {ClientColumnGettersByColId} from 'app/client/models/ClientColumnGetters';
 import debounce = require('lodash/debounce');
 
 // Limit on the how many rows to request for OnDemand tables.
@@ -306,28 +305,9 @@ export class TableQuerySets {
 export function getFilterFunc(docData: DocData, query: ClientQuery): RowFilterFunc<UIRowId> {
   // NOTE we rely without checking on tableId and colIds being valid.
   const tableData: BaseTableData = docData.getTable(query.tableId)!;
-  const colFuncs = Object.keys(query.filters).sort().map(
-    (colId) => {
-      const getter = tableData.getRowPropFunc(colId)!;
-      const values = new Set(query.filters[colId]);
-      switch (query.operations[colId]) {
-        case "intersects":
-          return (rowId: UIRowId) => {
-            const value = getter(rowId) as CellValue;
-            return isList(value) &&
-              (decodeObject(value) as unknown[]).some(v => values.has(v));
-          };
-        case "empty":
-          return (rowId: UIRowId) => {
-            const value = getter(rowId);
-            // `isList(value) && value.length === 1` means `value == ['L']` i.e. an empty list
-            return !value || isList(value) && value.length === 1;
-          };
-        case "in":
-          return (rowId: UIRowId) => values.has(getter(rowId));
-      }
-    });
-  return (rowId: UIRowId) => colFuncs.every(f => f(rowId));
+  const colGetters = new ClientColumnGettersByColId(tableData);
+  const rowFilterFunc = getLinkingFilterFunc(colGetters, query);
+  return (rowId: UIRowId) => rowId !== "new" && rowFilterFunc(rowId);
 }
 
 /**

app/client/models/entities/ViewSectionRec.ts

@@ -1,6 +1,6 @@
 import BaseView from 'app/client/components/BaseView';
 import {CursorPos} from 'app/client/components/Cursor';
-import {FilterColValues, LinkingState} from 'app/client/components/LinkingState';
+import {LinkingState} from 'app/client/components/LinkingState';
 import {KoArray} from 'app/client/lib/koArray';
 import {
   ColumnRec,
@@ -17,6 +17,7 @@ import {
 import * as modelUtil from 'app/client/models/modelUtil';
 import {LinkConfig} from 'app/client/ui/selectBy';
 import {getWidgetTypes} from 'app/client/ui/widgetTypes';
+import {FilterColValues} from "app/common/ActiveDocAPI";
 import {AccessLevel, ICustomWidget} from 'app/common/CustomWidget';
 import {UserAction} from 'app/common/DocActions';
 import {arrayRepeat} from 'app/common/gutil';

app/common/ActiveDocAPI.ts

@@ -116,6 +116,8 @@ export interface ClientQuery extends BaseQuery {
   };
 }
 
+export type FilterColValues = Pick<ClientQuery, "filters" | "operations">;
+
 /**
  * Query intended to be sent to a server.
  */

app/common/ColumnGetters.ts

@@ -26,4 +26,12 @@ export interface ColumnGetters {
   getManualSortGetter(): ColumnGetter | null;
 }
 
+/**
+ * Like ColumnGetters, but takes the string `colId` rather than a `ColSpec`
+ * or numeric row ID.
+ */
+export interface ColumnGettersByColId {
+  getColGetterByColId(colId: string): ColumnGetter | null;
+}
+
 export type ColumnGetter = (rowId: number) => any;

app/common/RowFilterFunc.ts

@@ -1,5 +1,9 @@
-import { CellValue } from "app/common/DocActions";
-import { ColumnFilterFunc } from "app/common/ColumnFilterFunc";
+import {CellValue} from "app/common/DocActions";
+import {ColumnFilterFunc} from "app/common/ColumnFilterFunc";
+import {FilterColValues} from 'app/common/ActiveDocAPI';
+import {isList} from 'app/common/gristTypes';
+import {decodeObject} from 'app/plugin/objtypes';
+import {ColumnGettersByColId} from 'app/common/ColumnGetters';
 
 export type RowFilterFunc<T> = (row: T) => boolean;
 
@@ -14,3 +18,32 @@ export function buildRowFilter<T>(
 }
 
 export type RowValueFunc<T> = (rowId: T) => CellValue;
+
+// Filter rows for the purpose of linked widgets
+export function getLinkingFilterFunc(
+  columnGetters: ColumnGettersByColId, {filters, operations}: FilterColValues
+): RowFilterFunc<number> {
+  const colFuncs = Object.keys(filters).sort().map(
+    (colId) => {
+      const getter = columnGetters.getColGetterByColId(colId);
+      if (!getter) { return () => true; }
+      const values = new Set(filters[colId]);
+      switch (operations[colId]) {
+        case "intersects":
+          return (rowId: number) => {
+            const value = getter(rowId) as CellValue;
+            return isList(value) &&
+              (decodeObject(value) as unknown[]).some(v => values.has(v));
+          };
+        case "empty":
+          return (rowId: number) => {
+            const value = getter(rowId);
+            // `isList(value) && value.length === 1` means `value == ['L']` i.e. an empty list
+            return !value || isList(value) && value.length === 1;
+          };
+        case "in":
+          return (rowId: number) => values.has(getter(rowId));
+      }
+    });
+  return (rowId: number) => colFuncs.every(f => f(rowId));
+}

app/server/lib/Export.ts

@@ -1,3 +1,4 @@
+import {FilterColValues} from 'app/common/ActiveDocAPI';
 import {ApiError} from 'app/common/ApiError';
 import {buildColFilter} from 'app/common/ColumnFilterFunc';
 import {TableDataAction, TableDataActionSet} from 'app/common/DocActions';
@@ -7,7 +8,7 @@ import * as gristTypes from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
 import {nativeCompare} from 'app/common/gutil';
 import {isTableCensored} from 'app/common/isHiddenTable';
-import {buildRowFilter} from 'app/common/RowFilterFunc';
+import {buildRowFilter, getLinkingFilterFunc} from 'app/common/RowFilterFunc';
 import {schema, SchemaTypes} from 'app/common/schema';
 import {SortFunc} from 'app/common/SortFunc';
 import {Sort} from 'app/common/SortSpec';
@@ -97,6 +98,7 @@ export interface ExportParameters {
   viewSectionId?: number;
   sortOrder?: number[];
   filters?: Filter[];
+  linkingFilter?: FilterColValues;
 }
 
 /**
@@ -114,12 +116,14 @@ export function parseExportParameters(req: express.Request): ExportParameters {
   const viewSectionId = optIntegerParam(req.query.viewSection);
   const sortOrder = optJsonParam(req.query.activeSortSpec, []) as number[];
   const filters: Filter[] = optJsonParam(req.query.filters, []);
+  const linkingFilter: FilterColValues = optJsonParam(req.query.linkingFilter, null);
 
   return {
     tableId,
     viewSectionId,
     sortOrder,
     filters,
+    linkingFilter,
   };
 }
 
@@ -268,11 +272,12 @@ export async function exportSection(
   viewSectionId: number,
   sortSpec: Sort.SortSpec | null,
   filters: Filter[] | null,
+  linkingFilter: FilterColValues | null = null,
   req: express.Request,
   {metaTables}: {metaTables?: TableDataActionSet} = {},
 ): Promise<ExportData> {
   return doExportSection(new ActiveDocSourceDirect(activeDoc, req), viewSectionId, sortSpec,
-    filters, {metaTables});
+    filters, linkingFilter, {metaTables});
 }
 
 export async function doExportSection(
@@ -280,6 +285,7 @@ export async function doExportSection(
   viewSectionId: number,
   sortSpec: Sort.SortSpec | null,
   filters: Filter[] | null,
+  linkingFilter: FilterColValues | null = null,
   {metaTables}: {metaTables?: TableDataActionSet} = {},
 ): Promise<ExportData> {
   metaTables = metaTables || await getMetaTables(activeDocSource);
@@ -360,6 +366,10 @@ export async function doExportSection(
   // filter rows numbers
   rowIds = rowIds.filter(rowFilter);
 
+  if (linkingFilter) {
+    rowIds = rowIds.filter(getLinkingFilterFunc(getters, linkingFilter));
+  }
+
   const docInfo = safeRecord(safeTable(metaTables, '_grist_DocInfo'), 1);
   const docSettings = gutil.safeJsonParse(docInfo.documentSettings, {});
 

app/server/lib/ExportCSV.ts

@@ -1,5 +1,6 @@
 import {ApiError} from 'app/common/ApiError';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
+import {FilterColValues} from "app/common/ActiveDocAPI";
 import {DownloadOptions, ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export';
 import log from 'app/server/lib/log';
 import * as bluebird from 'bluebird';
@@ -16,9 +17,10 @@ bluebird.promisifyAll(csv);
 export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
                                   res: express.Response, options: DownloadOptions) {
   log.info('Generating .csv file...');
-  const {filename, tableId, viewSectionId, filters, sortOrder} = options;
+  const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
   const data = viewSectionId ?
-    await makeCSVFromViewSection(activeDoc, viewSectionId, sortOrder || null, filters || null, req) :
+    await makeCSVFromViewSection(
+      activeDoc, viewSectionId, sortOrder || null, filters || null, linkingFilter || null, req) :
     await makeCSVFromTable(activeDoc, tableId, req);
   res.set('Content-Type', 'text/csv');
   res.setHeader('Content-Disposition', contentDisposition(filename + '.csv'));
@@ -41,9 +43,10 @@ export async function makeCSVFromViewSection(
   viewSectionId: number,
   sortOrder: number[] | null,
   filters: Filter[] | null,
+  linkingFilter: FilterColValues | null,
   req: express.Request) {
 
-  const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, req);
+  const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, linkingFilter, req);
   const file = convertToCsv(data);
   return file;
 }

app/server/lib/ExportXLSX.ts

@@ -55,7 +55,7 @@ export async function downloadXLSX(activeDoc: ActiveDoc, req: express.Request,
 export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request,
                                  outputStream: Writable, options: ExportParameters) {
   log.debug(`Generating .xlsx file`);
-  const {tableId, viewSectionId, filters, sortOrder} = options;
+  const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
   const testDates = (req.hostname === 'localhost');
 
   const { port1, port2 } = new MessageChannel();
@@ -90,7 +90,7 @@ export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request,
     // hanlding 3 cases : full XLSX export (full file), view xlsx export, table xlsx export
     try {
       if (viewSectionId) {
-        await run('makeXLSXFromViewSection', viewSectionId, sortOrder, filters);
+        await run('makeXLSXFromViewSection', viewSectionId, sortOrder, filters, linkingFilter);
       } else if (tableId) {
         await run('makeXLSXFromTable', tableId);
       } else {

app/server/lib/ServerColumnGetters.ts

@@ -1,8 +1,9 @@
-import { ColumnGetter, ColumnGetters } from 'app/common/ColumnGetters';
+import {ColumnGetter, ColumnGetters, ColumnGettersByColId} from 'app/common/ColumnGetters';
 import * as gristTypes from 'app/common/gristTypes';
-import { safeJsonParse } from 'app/common/gutil';
-import { choiceGetter } from 'app/common/SortFunc';
-import { Sort } from 'app/common/SortSpec';
+import {safeJsonParse} from 'app/common/gutil';
+import {choiceGetter} from 'app/common/SortFunc';
+import {Sort} from 'app/common/SortSpec';
+import {BulkColValues} from 'app/plugin/GristData';
 
 /**
  *
@@ -10,11 +11,11 @@ import { Sort } from 'app/common/SortSpec';
  * drawing on the data and metadata prepared for CSV export.
  *
  */
-export class ServerColumnGetters implements ColumnGetters {
+export class ServerColumnGetters implements ColumnGetters, ColumnGettersByColId {
   private _rowIndices: Map<number, number>;
   private _colIndices: Map<number, string>;
 
-  constructor(rowIds: number[], private _dataByColId: {[colId: string]: any}, private _columns: any[]) {
+  constructor(rowIds: number[], private _dataByColId: BulkColValues, private _columns: any[]) {
     this._rowIndices = new Map<number, number>(rowIds.map((rowId, index) => [rowId, index] as [number, number]));
     this._colIndices = new Map<number, string>(_columns.map(col => [col.id, col.colId] as [number, string]));
   }
@@ -25,14 +26,10 @@ export class ServerColumnGetters implements ColumnGetters {
     if (colId === undefined) {
       return null;
     }
-    const col = this._dataByColId[colId];
-    let getter = (rowId: number) => {
-      const idx = this._rowIndices.get(rowId);
-      if (idx === undefined) {
-        return null;
-      }
-      return col[idx];
-    };
+    let getter = this.getColGetterByColId(colId);
+    if (!getter) {
+      return null;
+    }
     const details = Sort.specToDetails(colSpec);
     if (details.orderByChoice) {
       const rowModel = this._columns.find(c => c.id == colRef);
@@ -51,4 +48,22 @@ export class ServerColumnGetters implements ColumnGetters {
     }
     return this.getColGetter(manualSortCol.id);
   }
+
+  public getColGetterByColId(colId: string): ColumnGetter | null {
+    if (colId === "id") {
+      return (rowId: number) => rowId;
+    }
+    const col = this._dataByColId[colId];
+    if (!col) {
+      return null;
+    }
+    return (rowId: number) => {
+      const idx = this._rowIndices.get(rowId);
+      if (idx === undefined) {
+        return null;
+      }
+      return col[idx];
+    };
+  }
+
 }

app/server/lib/workerExporter.ts

@@ -1,4 +1,5 @@
 import {PassThrough} from 'stream';
+import {FilterColValues} from "app/common/ActiveDocAPI";
 import {ActiveDocSource, doExportDoc, doExportSection, doExportTable, ExportData, Filter} from 'app/server/lib/Export';
 import {createExcelFormatter} from 'app/server/lib/ExcelFormatter';
 import * as log from 'app/server/lib/log';
@@ -87,8 +88,9 @@ async function doMakeXLSXFromViewSection(
   viewSectionId: number,
   sortOrder: number[],
   filters: Filter[],
+  linkingFilter: FilterColValues,
 ) {
-  const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters);
+  const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters, linkingFilter);
   const {exportTable, end} = convertToExcel(stream, testDates);
   exportTable(data);
   await end();

test/nbrowser/SelectByRefList.ts

@@ -198,6 +198,12 @@ async function checkSelectingRecords(selectBy: string, sourceData: string[][], n
       }),
       sourceGroup
     );
+    const csvCells = await gu.downloadSectionCsvGridCells('LINKTARGET');
+    const expectedCsvCells = sourceGroup.slice(0, -3)  // remove 'add new' row of empty strings
+      // visible cells text uses newlines to separate list items,
+      // CSV export uses commas
+      .map(s => s.replace("\n", ", "));
+    assert.deepEqual(csvCells, expectedCsvCells);
   }
 
   for (let i = 0; i < sourceData.length; i++) {

test/nbrowser/SelectBySummary.ts

@@ -232,9 +232,6 @@ async function checkSelectingRecords(
     const targetGroup = targetData[targetGroupIndex];
     const countCell = await gu.getCell({section: summarySection, col: 'count', rowNum: targetGroupIndex + 1});
     const numTargetRows = targetGroup.length / 3;
-    if (targetSection === 'TABLE1') {
-      assert.equal(await countCell.getText(), numTargetRows.toString());
-    }
     await countCell.click();
     assert.deepEqual(
       await gu.getVisibleGridCells({
@@ -244,6 +241,13 @@ async function checkSelectingRecords(
       }),
       targetGroup
     );
+    if (targetSection === 'TABLE1') {
+      assert.equal(await countCell.getText(), numTargetRows.toString());
+      const csvCells = await gu.downloadSectionCsvGridCells(targetSection);
+      // visible cells text uses newlines to separate list items, CSV export uses commas
+      const expectedCsvCells = targetGroup.map(s => s.replace("\n", ", "));
+      assert.deepEqual(csvCells, expectedCsvCells);
+    }
   }
 
   for (let i = 0; i < targetData.length; i++) {

test/nbrowser/gristUtils.ts

@@ -11,6 +11,7 @@ import { assert, driver as driverOrig, error, Key, WebElement, WebElementPromise
 import { stackWrapFunc, stackWrapOwnMethods, WebDriver } from 'mocha-webdriver';
 import * as path from 'path';
 
+import {csvDecodeRow} from 'app/common/csvFormat';
 import { decodeUrl } from 'app/common/gristUrls';
 import { FullUser, UserProfile } from 'app/common/LoginSessionAPI';
 import { resetOrg } from 'app/common/resetOrg';
@@ -29,6 +30,7 @@ import { server } from 'test/nbrowser/testServer';
 import { Cleanup } from 'test/nbrowser/testUtils';
 import * as testUtils from 'test/server/testUtils';
 import type { AssertionError } from 'assert';
+import axios from 'axios';
 
 // tslint:disable:no-namespace
 // Wrap in a namespace so that we can apply stackWrapOwnMethods to all the exports together.
@@ -3078,6 +3080,24 @@ export function produceUncaughtError(message: string) {
   }, message);
 }
 
+export async function downloadSectionCsv(
+  section: string, headers: any = {Authorization: 'Bearer api_key_for_chimpy'}
+) {
+  await openSectionMenu("viewLayout", section);
+  const href = await driver.findWait('.test-download-section', 1000).getAttribute('href');
+  await driver.sendKeys(Key.ESCAPE);  // Close section menu
+  const resp = await axios.get(href, { responseType: 'text', headers });
+  return resp.data as string;
+}
+
+export async function downloadSectionCsvGridCells(
+  section: string, headers: any = {Authorization: 'Bearer api_key_for_chimpy'}
+): Promise<string[]> {
+  const csvString = await downloadSectionCsv(section, headers);
+  const csvRows = csvString.split('\n').slice(1).map(csvDecodeRow);
+  return ([] as string[]).concat(...csvRows);
+}
+
 } // end of namespace gristUtils
 
 stackWrapOwnMethods(gristUtils);