gristlabs_grist-core/app/client/models/QuerySet.ts
Alex Hall 7f1f8fc9e6 (core) Linking summary tables grouped by list columns
Summary:
Prefix keys of `LinkingState.filterColValues` with `_contains:` when the source column is a ChoiceList or ReferenceList.

This is parsed out to make a boolean `isContainsFilter` which is kept in each value of `QueryRefs.filterTuples` (previously `filterPairs`).

Then when converting back in `convertQueryFromRefs` we construct `Query.contains: {[colId: string]: boolean}`.

Finally `getFilterFunc` uses `Query.contains` to decide what kind of filtering to do.

This is not pretty, but the existing code is already very complex and it was hard to find something that wouldn't require touching loads of code just to make things compile.

Test Plan: Added a new nbrowser test and fixture, tests that selecting a source table by summary tables grouped by a choicelist column, non-list column, and both all filter the correct data.

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2940
2021-08-10 20:41:24 +02:00

395 lines
16 KiB
TypeScript

/**
* A QuerySet represents a data query to the server, which returns matching data and includes a
* subscription. The subscription tells the server to send us docActions that affect this query.
*
* This file combines several classes related to it:
*
* - QuerySetManager is maintained by GristDoc, and keeps all active QuerySets for this doc.
* A new one is created using QuerySetManager.useQuerySet(owner, query)
*
* This creates a subscription to the server, and sets up owner.autoDispose() to clean up
* that subscription. If a subscription already exists, it only returns a reference to it,
* and disposal will remove the reference, only unsubscribing from the server when no
* referernces remain.
*
* - DynamicQuerySet is used by BaseView (in place of FilteredRowSource used previously). It is a
* single RowSource which mirrors a QuerySet, and allows the QuerySet to be changed.
* You set it to a new query using DynamicQuerySet.makeQuery(...)
*
* - QuerySet represents the actual query, makes the calls to the server to populate the data in
* the relevant TableData. It is also a FilteredRowSource for the rows matching the query.
*
* - TableQuerySets is a simple set of queries maintained for a single table (by DataTableModel).
* It's needed to know which rows are still relevant after a QuerySet is disposed.
*
* TODO: need to have a fetch limit (e.g. 1000 by default, or an option for user)
* TODO: client-side should show "..." or "50000 more rows not shown" in that case.
* TODO: Reference columns don't work properly because always use a displayCol which relies on formulas
*/
import * as DataTableModel from 'app/client/models/DataTableModel';
import {DocModel} from 'app/client/models/DocModel';
import {BaseFilteredRowSource, RowId, RowList, RowSource} from 'app/client/models/rowset';
import {TableData} from 'app/client/models/TableData';
import {ActiveDocAPI, ClientQuery, QueryOperation} from 'app/common/ActiveDocAPI';
import {CellValue, TableDataAction} from 'app/common/DocActions';
import {DocData} from 'app/common/DocData';
import {nativeCompare} from 'app/common/gutil';
import {IRefCountSub, RefCountMap} from 'app/common/RefCountMap';
import {TableData as BaseTableData} from 'app/common/TableData';
import {RowFilterFunc} from 'app/common/RowFilterFunc';
import {tbind} from 'app/common/tbind';
import {Disposable, Holder, IDisposableOwnerT} from 'grainjs';
import * as ko from 'knockout';
import debounce = require('lodash/debounce');
import {isList} from "app/common/gristTypes";
import {decodeObject} from "app/plugin/objtypes";
// Limit on the how many rows to request for OnDemand tables.
const ON_DEMAND_ROW_LIMIT = 10000;
// Copied from app/server/lib/DocStorage.js. Actually could be 999, we are just playing it safe.
const MAX_SQL_PARAMS = 500;
/**
* A representation of a Query that uses tableRef/colRefs (i.e. metadata rowIds) to remain stable
* across table/column renames.
*/
export interface QueryRefs {
tableRef: number;
filterTuples: Array<FilterTuple>;
}
type FilterTuple = [number, QueryOperation, any[]];
/**
* QuerySetManager keeps track of all queries for a GristDoc instance. It is also responsible for
* disposing all state associated with queries when a GristDoc is disposed.
*
* Note that queries are made using tableId + colIds, which is a more suitable interface for a
* (future) public API, and easier to interact with DocData/TableData. However, it creates
* problems when tables or columns are renamed or deleted.
*
* To handle renames, we keep track of queries using their QueryRef representation, using
* tableRef/colRefs, i.e. metadata rowIds that aren't affected by renames.
*
* To handle deletes, we subscribe to isDeleted() observables of the needed tables and columns,
* and purge the query from QuerySetManager if any isDeleted() flag becomes true.
*/
export class QuerySetManager extends Disposable {
private _queryMap: RefCountMap<string, QuerySet>;
constructor(private _docModel: DocModel, docComm: ActiveDocAPI) {
super();
this._queryMap = this.autoDispose(new RefCountMap<string, QuerySet>({
create: (query: string) => QuerySet.create(null, _docModel, docComm, query, this),
dispose: (query: string, querySet: QuerySet) => querySet.dispose(),
gracePeriodMs: 60000, // Dispose after a minute of disuse.
}));
}
public useQuerySet(owner: IDisposableOwnerT<IRefCountSub<QuerySet>>, query: ClientQuery): QuerySet {
// Convert the query to a string key which identifies it.
const queryKey: string = encodeQuery(convertQueryToRefs(this._docModel, query));
// Look up or create the query in the RefCountMap. The returned object is a RefCountSub
// subscription, which decrements reference count when disposed.
const querySetRefCount = this._queryMap.use(queryKey);
// The passed-in owner is what will dispose this subscription (decrement reference count).
owner.autoDispose(querySetRefCount);
return querySetRefCount.get();
}
public purgeKey(queryKey: string) {
this._queryMap.purgeKey(queryKey);
}
// For testing: set gracePeriodMs, returning the previous value.
public testSetGracePeriodMs(ms: number): number {
return this._queryMap.testSetGracePeriodMs(ms);
}
}
/**
* DynamicQuerySet wraps one QuerySet, and allows changing it on the fly. It serves as a
* RowSource.
*/
export class DynamicQuerySet extends RowSource {
// Holds a reference to the currently active QuerySet.
private _holder = Holder.create<IRefCountSub<QuerySet>>(this);
// Shortcut to _holder.get().get().
private _querySet?: QuerySet;
// We could switch between several different queries quickly. If several queries are done
// fetching at the same time (e.g. were already ready), debounce lets us only update the
// query-set once to the last query.
private _updateQuerySetDebounced = debounce(tbind(this._updateQuerySet, this), 0);
constructor(private _querySetManager: QuerySetManager, private _tableModel: DataTableModel) {
super();
}
public getAllRows(): RowList {
return this._querySet ? this._querySet.getAllRows() : [];
}
public getNumRows(): number {
return this._querySet ? this._querySet.getNumRows() : 0;
}
/**
* Tells whether the query's result got truncated, i.e. not all rows are included.
*/
public get isTruncated(): boolean {
return this._querySet ? this._querySet.isTruncated : false;
}
/**
* Replace the query represented by this DynamicQuerySet. If multiple makeQuery() calls are made
* quickly (while waiting for the server), cb() may only be called for the latest one.
*
* If there is an error fetching data, cb(err) will be called with that error. The second
* argument to cb() is true if any data was changed, and false if not. Note that for a series of
* makeQuery() calls, cb() is always called at least once, and always asynchronously.
*/
public makeQuery(filters: {[colId: string]: any[]},
operations: {[colId: string]: QueryOperation},
cb: (err: Error|null, changed: boolean) => void): void {
const query: ClientQuery = {tableId: this._tableModel.tableData.tableId, filters, operations};
const newQuerySet = this._querySetManager.useQuerySet(this._holder, query);
// CB should be called asynchronously, since surprising hard-to-debug interactions can happen
// if it's sometimes synchronous and sometimes not.
newQuerySet.fetchPromise.then(() => {
this._updateQuerySetDebounced(newQuerySet, cb);
})
.catch((err) => { cb(err, false); });
}
private _updateQuerySet(nextQuerySet: QuerySet, cb: (err: Error|null, changed: boolean) => void): void {
try {
if (nextQuerySet !== this._querySet) {
const oldQuerySet = this._querySet;
this._querySet = nextQuerySet;
if (oldQuerySet) {
this.stopListening(oldQuerySet, 'rowChange');
this.stopListening(oldQuerySet, 'rowNotify');
this.trigger('rowChange', 'remove', oldQuerySet.getAllRows());
}
this.trigger('rowChange', 'add', this._querySet.getAllRows());
this.listenTo(this._querySet, 'rowNotify', tbind(this.trigger, this, 'rowNotify'));
this.listenTo(this._querySet, 'rowChange', tbind(this.trigger, this, 'rowChange'));
}
cb(null, true);
} catch (err) {
cb(err, true);
}
}
}
/**
* Class representing a query, which knows how to fetch the data, an presents a RowSource with
* matching rows. It uses new Comm calls for onDemand tables, but for regular tables, fetching
* data uses the good old tableModel.fetch(). In in most cases the data is already available, so
* this class is little more than a FilteredRowSource.
*/
export class QuerySet extends BaseFilteredRowSource {
// A publicly exposed promise, which may be waited on in order to know that the data has
// arrived. Until then, the RowSource underlying this QuerySet is empty.
public readonly fetchPromise: Promise<void>;
// Whether the fetched result is considered incomplete, i.e. not all rows were fetched.
public isTruncated: boolean;
constructor(docModel: DocModel, docComm: ActiveDocAPI, queryKey: string, qsm: QuerySetManager) {
const queryRefs: QueryRefs = decodeQuery(queryKey);
const query: ClientQuery = convertQueryFromRefs(docModel, queryRefs);
super(getFilterFunc(docModel.docData, query));
this.isTruncated = false;
// When table or any needed columns are deleted, purge this QuerySet from the map.
const isInvalid = this.autoDispose(makeQueryInvalidComputed(docModel, queryRefs));
this.autoDispose(isInvalid.subscribe((invalid) => {
if (invalid) { qsm.purgeKey(queryKey); }
}));
// Find the relevant DataTableModel.
const tableModel = docModel.dataTables[query.tableId];
// The number of values across all filters is limited to MAX_SQL_PARAMS. Normally a query has
// a single filter column, but in case there are multiple we divide the limit across all
// columns. It's OK to modify the query in place, since this modified version is not used
// elsewhere.
// (It might be better to limit this in DocStorage.js, but by limiting here, it's easier to
// know when to set isTruncated flag, to inform the user that data is incomplete.)
const colIds = Object.keys(query.filters);
if (colIds.length > 0) {
const maxParams = Math.floor(MAX_SQL_PARAMS / colIds.length);
for (const c of colIds) {
const values = query.filters[c];
if (values.length > maxParams) {
query.filters[c] = values.slice(0, maxParams);
this.isTruncated = true;
}
}
}
let fetchPromise: Promise<void>;
if (tableModel.tableMetaRow.onDemand()) {
const tableQS = tableModel.tableQuerySets;
fetchPromise = docComm.useQuerySet({limit: ON_DEMAND_ROW_LIMIT, ...query}).then((data) => {
// We assume that if we fetched the max number of rows, that there are likely more and the
// result should be reported as truncated.
// TODO: Better to fetch ON_DEMAND_ROW_LIMIT + 1 and omit one of them, so that isTruncated
// is only set if the row limit really was exceeded.
const rowIds = data.tableData[2];
if (rowIds.length >= ON_DEMAND_ROW_LIMIT) {
this.isTruncated = true;
}
this.onDispose(() => {
docComm.disposeQuerySet(data.querySubId).catch((err) => {
// tslint:disable-next-line:no-console
console.log(`Promise rejected for disposeQuerySet: ${err.message}`);
});
tableQS.removeQuerySet(this);
});
tableQS.addQuerySet(this, data.tableData);
});
} else {
// For regular (small), we fetch in bulk (and do nothing if already fetched).
fetchPromise = tableModel.fetch(false);
}
// This is a FilteredRowSource; subscribe it to the underlying data once the fetch resolves.
this.fetchPromise = fetchPromise.then(() => this.subscribeTo(tableModel));
}
}
/**
* Helper for use in a DataTableModel to maintain all QuerySets.
*/
export class TableQuerySets {
private _querySets: Set<QuerySet> = new Set();
constructor(private _tableData: TableData) {}
public addQuerySet(querySet: QuerySet, data: TableDataAction): void {
this._querySets.add(querySet);
this._tableData.loadPartial(data);
}
// Returns a Set of unused RowIds from querySet.
public removeQuerySet(querySet: QuerySet): void {
this._querySets.delete(querySet);
// Figure out which rows are not used by any other QuerySet in this DataTableModel.
const unusedRowIds = new Set(querySet.getAllRows());
for (const qs of this._querySets) {
for (const rowId of qs.getAllRows()) {
unusedRowIds.delete(rowId);
}
}
this._tableData.unloadPartial(Array.from(unusedRowIds) as number[]);
}
}
/**
* Returns a filtering function which tells whether a row matches the given query.
*/
export function getFilterFunc(docData: DocData, query: ClientQuery): RowFilterFunc<RowId> {
// NOTE we rely without checking on tableId and colIds being valid.
const tableData: BaseTableData = docData.getTable(query.tableId)!;
const colFuncs = Object.keys(query.filters).sort().map(
(colId) => {
const getter = tableData.getRowPropFunc(colId)!;
const values = new Set(query.filters[colId]);
switch (query.operations![colId]) {
case "intersects":
return (rowId: RowId) => {
const value = getter(rowId) as CellValue;
return isList(value) &&
(decodeObject(value) as unknown[]).some(v => values.has(v));
};
case "in":
return (rowId: RowId) => values.has(getter(rowId));
default:
throw new Error("Unknown operation");
}
});
return (rowId: RowId) => colFuncs.every(f => f(rowId));
}
/**
* Helper that converts a Query (with tableId/colIds) to an object with tableRef/colRefs (i.e.
* rowIds), and consistently sorted. We use that to identify a Query across table/column renames.
*/
function convertQueryToRefs(docModel: DocModel, query: ClientQuery): QueryRefs {
const tableRec: any = docModel.dataTables[query.tableId].tableMetaRow;
const colRefsByColId: {[colId: string]: number} = {};
for (const col of tableRec.columns.peek().peek()) {
colRefsByColId[col.colId.peek()] = col.getRowId();
}
const filterTuples = Object.keys(query.filters).map((colId) => {
const values = query.filters[colId];
// Keep filter values sorted by value, for consistency.
values.sort(nativeCompare);
return [colRefsByColId[colId], query.operations![colId], values] as FilterTuple;
});
// Keep filters sorted by colRef, for consistency.
filterTuples.sort((a, b) =>
nativeCompare(a[0], b[0]) || nativeCompare(a[1], b[1]));
return {tableRef: tableRec.getRowId(), filterTuples};
}
/**
* Helper to convert a QueryRefs (using tableRef/colRefs) object back to a Query (using
* tableId/colIds).
*/
function convertQueryFromRefs(docModel: DocModel, queryRefs: QueryRefs): ClientQuery {
const tableRec = docModel.dataTablesByRef.get(queryRefs.tableRef)!.tableMetaRow;
const filters: {[colId: string]: any[]} = {};
const operations: {[colId: string]: QueryOperation} = {};
for (const [colRef, operation, values] of queryRefs.filterTuples) {
const colId = docModel.columns.getRowModel(colRef).colId.peek();
filters[colId] = values;
operations[colId] = operation;
}
return {tableId: tableRec.tableId.peek(), filters, operations};
}
/**
* Encodes a query (converted to QueryRefs using convertQueryToRefs()) as a string, to be usable
* as a key into a map.
*
* It uses JSON.stringify, but avoids objects since their order of keys in serialization is not
* guaranteed. This is important to produce consistent results (same query => same encoding).
*/
function encodeQuery(queryRefs: QueryRefs): string {
return JSON.stringify([queryRefs.tableRef, queryRefs.filterTuples]);
}
// Decode an encoded QueryRefs.
function decodeQuery(queryKey: string): QueryRefs {
const [tableRef, filterTuples] = JSON.parse(queryKey);
return {tableRef, filterTuples};
}
/**
* Returns a ko.computed() which turns to true when the table or any of the columns needed by the
* given query are deleted.
*/
function makeQueryInvalidComputed(docModel: DocModel, queryRefs: QueryRefs): ko.Computed<boolean> {
const tableFlag: ko.Observable<boolean> = docModel.tables.getRowModel(queryRefs.tableRef)._isDeleted;
const colFlags: Array<ko.Observable<boolean>> = queryRefs.filterTuples.map(
([colRef, , ]) => docModel.columns.getRowModel(colRef)._isDeleted);
return ko.computed(() => Boolean(tableFlag() || colFlags.some((c) => c())));
}