From 7fe4423a6f5ac9a63808d81c850d6528d2a99479 Mon Sep 17 00:00:00 2001 From: George Gevoian Date: Fri, 19 Nov 2021 12:30:11 -0800 Subject: [PATCH] (core) Allow filtering hidden columns Summary: Existing filters are now moved out of fields and into a new metadata table for filters, and the client is updated to retrieve/update/save filters from the new table. This enables storing of filters for columns that don't have fields (notably, hidden columns). Test Plan: Browser and server tests. Reviewers: jarek Reviewed By: jarek Differential Revision: https://phab.getgrist.com/D3138 --- app/client/components/BaseView.js | 9 +- app/client/components/GridView.js | 4 +- app/client/components/GristDoc.ts | 6 +- app/client/components/ViewConfigTab.js | 21 ++- app/client/declarations.d.ts | 5 +- app/client/models/DocModel.ts | 3 + app/client/models/SectionFilter.ts | 39 ++-- app/client/models/entities/ColumnRec.ts | 31 ++++ app/client/models/entities/FilterRec.ts | 22 +++ app/client/models/entities/ViewFieldRec.ts | 15 -- app/client/models/entities/ViewSectionRec.ts | 185 ++++++++++++++++--- app/client/ui/ColumnFilterMenu.ts | 44 +++-- app/client/ui/FilterBar.ts | 50 ++--- app/client/ui/ViewSectionMenu.ts | 45 +++-- app/common/schema.ts | 14 +- app/server/lib/Export.ts | 39 ++-- app/server/lib/initialDocSql.ts | 6 +- sandbox/grist/docmodel.py | 6 + sandbox/grist/migrations.py | 33 ++++ sandbox/grist/schema.py | 19 +- 20 files changed, 431 insertions(+), 165 deletions(-) create mode 100644 app/client/models/entities/FilterRec.ts diff --git a/app/client/components/BaseView.js b/app/client/components/BaseView.js index ae5d73f0..40d263db 100644 --- a/app/client/components/BaseView.js +++ b/app/client/components/BaseView.js @@ -78,7 +78,7 @@ function BaseView(gristDoc, viewSectionModel, options) { // Create a section filter and a filtered row source that subscribes to its changes. // `sectionFilter` also provides an `addTemporaryRow()` to allow views to display newly inserted rows, // and `setFilterOverride()` to allow controlling a filter from a column menu. - this._sectionFilter = SectionFilter.create(this, this.viewSection.viewFields, this.tableModel.tableData); + this._sectionFilter = SectionFilter.create(this, this.viewSection, this.tableModel.tableData); this._filteredRowSource = rowset.FilteredRowSource.create(this, this._sectionFilter.sectionFilterFunc.get()); this._filteredRowSource.subscribeTo(this._mainRowSource); this.autoDispose(this._sectionFilter.sectionFilterFunc.addListener(filterFunc => { @@ -669,10 +669,11 @@ BaseView.prototype.getLastDataRowIndex = function() { }; /** - * Creates and opens ColumnFilterMenu for a given field, and returns its PopupControl. + * Creates and opens ColumnFilterMenu for a given field/column, and returns its PopupControl. */ -BaseView.prototype.createFilterMenu = function(openCtl, field, onClose) { - return createFilterMenu(openCtl, this._sectionFilter, field, this._mainRowSource, this.tableModel.tableData, onClose); +BaseView.prototype.createFilterMenu = function(openCtl, filterInfo, onClose) { + return createFilterMenu(openCtl, this._sectionFilter, filterInfo, this._mainRowSource, + this.tableModel.tableData, onClose); }; /** diff --git a/app/client/components/GridView.js b/app/client/components/GridView.js index cdc81a92..5430ca51 100644 --- a/app/client/components/GridView.js +++ b/app/client/components/GridView.js @@ -1386,7 +1386,9 @@ GridView.prototype._getColumnMenuOptions = function(copySelection) { GridView.prototype._columnFilterMenu = function(ctl, field) { this.ctxMenuHolder.autoDispose(ctl); - return this.createFilterMenu(ctl, field); + const filterInfo = this.viewSection.filters() + .find(({fieldOrColumn}) => fieldOrColumn.origCol().origColRef() === field.column().origColRef()); + return this.createFilterMenu(ctl, filterInfo); }; GridView.prototype.maybeSelectColumn = function (elem, field) { diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index e89d0deb..573b47ce 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -650,9 +650,9 @@ export class GristDoc extends DisposableWithEvents { } public getCsvLink() { - const filters = this.viewModel.activeSection.peek().filteredFields.get().map(field=> ({ - colRef : field.colRef.peek(), - filter : field.activeFilter.peek() + const filters = this.viewModel.activeSection.peek().activeFilters.get().map(filterInfo => ({ + colRef : filterInfo.fieldOrColumn.origCol().origColRef(), + filter : filterInfo.filter() })); const params = { diff --git a/app/client/components/ViewConfigTab.js b/app/client/components/ViewConfigTab.js index f50e8d74..5b51fc7d 100644 --- a/app/client/components/ViewConfigTab.js +++ b/app/client/components/ViewConfigTab.js @@ -576,29 +576,34 @@ ViewConfigTab.prototype._buildFilterDom = function() { } return [ - grainjsDom.forEach(section.filteredFields, (field) => { + grainjsDom.forEach(section.activeFilters, (filterInfo) => { return cssRow( cssIconWrapper( cssFilterIcon('FilterSimple', cssNoMarginLeft.cls('')), - attachColumnFilterMenu(section, field, { + attachColumnFilterMenu(section, filterInfo, { placement: 'bottom-end', attach: 'body', - trigger: ['click', (_el, popupControl) => popupControls.set(field, popupControl)], + trigger: [ + 'click', + (_el, popupControl) => popupControls.set(filterInfo.fieldOrColumn.origCol(), popupControl) + ], }), ), - cssLabel(grainjsDom.text(field.label)), + cssLabel(grainjsDom.text(filterInfo.fieldOrColumn.label)), cssIconWrapper( - cssFilterIcon('Remove', dom.on('click', () => field.activeFilter('')), - testId('remove-filter')), + cssFilterIcon('Remove', + dom.on('click', () => section.setFilter(filterInfo.fieldOrColumn.origCol().origColRef(), '')), + testId('remove-filter') + ), ), testId('filter'), ); }), cssRow( grainjsDom.domComputed((use) => { - const fields = use(use(section.viewFields).getObservable()); + const filters = use(section.filters); return cssTextBtn( cssPlusIcon('Plus'), 'Add Filter', - addFilterMenu(fields, popupControls, {placement: 'bottom-end'}), + addFilterMenu(filters, section, popupControls, {placement: 'bottom-end'}), testId('add-filter-btn'), ); }), diff --git a/app/client/declarations.d.ts b/app/client/declarations.d.ts index 59e17b84..a17cdf89 100644 --- a/app/client/declarations.d.ts +++ b/app/client/declarations.d.ts @@ -38,7 +38,8 @@ declare module "app/client/components/BaseView" { import {DataRowModel} from 'app/client/models/DataRowModel'; import {LazyArrayModel} from "app/client/models/DataTableModel"; import * as DataTableModel from "app/client/models/DataTableModel"; - import {ViewFieldRec, ViewSectionRec} from "app/client/models/DocModel"; + import {ViewSectionRec} from "app/client/models/DocModel"; + import {FilterInfo} from 'app/client/models/entities/ViewSectionRec'; import {SortedRowSet} from 'app/client/models/rowset'; import {FieldBuilder} from "app/client/widgets/FieldBuilder"; import {DomArg} from 'grainjs'; @@ -64,7 +65,7 @@ declare module "app/client/components/BaseView" { constructor(gristDoc: GristDoc, viewSectionModel: any); public setCursorPos(cursorPos: CursorPos): void; - public createFilterMenu(ctl: IOpenController, field: ViewFieldRec, onClose?: () => void): HTMLElement; + public createFilterMenu(ctl: IOpenController, filterInfo: FilterInfo, onClose?: () => void): HTMLElement; public buildTitleControls(): DomArg; public getLoadingDonePromise(): Promise; public activateEditorAtCursor(options?: Options): void; diff --git a/app/client/models/DocModel.ts b/app/client/models/DocModel.ts index 686c5740..2be93e81 100644 --- a/app/client/models/DocModel.ts +++ b/app/client/models/DocModel.ts @@ -28,6 +28,7 @@ import {schema, SchemaTypes} from 'app/common/schema'; import {ACLRuleRec, createACLRuleRec} from 'app/client/models/entities/ACLRuleRec'; import {ColumnRec, createColumnRec} from 'app/client/models/entities/ColumnRec'; import {createDocInfoRec, DocInfoRec} from 'app/client/models/entities/DocInfoRec'; +import {createFilterRec, FilterRec} from 'app/client/models/entities/FilterRec'; import {createPageRec, PageRec} from 'app/client/models/entities/PageRec'; import {createTabBarRec, TabBarRec} from 'app/client/models/entities/TabBarRec'; import {createTableRec, TableRec} from 'app/client/models/entities/TableRec'; @@ -41,6 +42,7 @@ import {createViewSectionRec, ViewSectionRec} from 'app/client/models/entities/V // import {ColumnRec, ViewFieldRec} from 'app/client/models/DocModel'; export {ColumnRec} from 'app/client/models/entities/ColumnRec'; export {DocInfoRec} from 'app/client/models/entities/DocInfoRec'; +export {FilterRec} from 'app/client/models/entities/FilterRec'; export {PageRec} from 'app/client/models/entities/PageRec'; export {TabBarRec} from 'app/client/models/entities/TabBarRec'; export {TableRec} from 'app/client/models/entities/TableRec'; @@ -111,6 +113,7 @@ export class DocModel { public validations: MTM = this._metaTableModel("_grist_Validations", createValidationRec); public pages: MTM = this._metaTableModel("_grist_Pages", createPageRec); public rules: MTM = this._metaTableModel("_grist_ACLRules", createACLRuleRec); + public filters: MTM = this._metaTableModel("_grist_Filters", createFilterRec); public docInfoRow: DocInfoRec; diff --git a/app/client/models/SectionFilter.ts b/app/client/models/SectionFilter.ts index 37d1a8c4..7306f522 100644 --- a/app/client/models/SectionFilter.ts +++ b/app/client/models/SectionFilter.ts @@ -1,6 +1,5 @@ -import {KoArray} from 'app/client/lib/koArray'; import {ColumnFilter} from 'app/client/models/ColumnFilter'; -import {ViewFieldRec} from 'app/client/models/DocModel'; +import {ColumnRec, ViewFieldRec, ViewSectionRec} from 'app/client/models/DocModel'; import {RowId} from 'app/client/models/rowset'; import {TableData} from 'app/client/models/TableData'; import {buildColFilter, ColumnFilterFunc} from 'app/common/ColumnFilterFunc'; @@ -10,15 +9,15 @@ import {Computed, Disposable, MutableObsArray, obsArray, Observable, UseCB} from export {ColumnFilterFunc} from 'app/common/ColumnFilterFunc'; interface OpenColumnFilter { - fieldRef: number; + colRef: number; colFilter: ColumnFilter; } -type ColFilterCB = (field: ViewFieldRec, colFilter: ColumnFilterFunc|null) => ColumnFilterFunc|null; +type ColFilterCB = (fieldOrColumn: ViewFieldRec|ColumnRec, colFilter: ColumnFilterFunc|null) => ColumnFilterFunc|null; /** * SectionFilter represents a collection of column filters in place for a view section. It is created - * out of `viewFields` and `tableData`, and provides a Computed `sectionFilterFunc` that users can + * out of `filters` (in `viewSection`) and `tableData`, and provides a Computed `sectionFilterFunc` that users can * subscribe to in order to update their FilteredRowSource. * * Additionally, `setFilterOverride()` provides a way to override the current filter for a given colRef, @@ -32,13 +31,13 @@ export class SectionFilter extends Disposable { private _openFilterOverride: Observable = Observable.create(this, null); private _tempRows: MutableObsArray = obsArray(); - constructor(public viewFields: ko.Computed>, private _tableData: TableData) { + constructor(public viewSection: ViewSectionRec, private _tableData: TableData) { super(); const columnFilterFunc = Computed.create(this, this._openFilterOverride, (use, openFilter) => { const openFilterFilterFunc = openFilter && use(openFilter.colFilter.filterFunc); - function getFilterFunc(field: ViewFieldRec, colFilter: ColumnFilterFunc|null) { - if (openFilter?.fieldRef === field.getRowId()) { + function getFilterFunc(fieldOrColumn: ViewFieldRec|ColumnRec, colFilter: ColumnFilterFunc|null) { + if (openFilter?.colRef === fieldOrColumn.getRowId()) { return openFilterFilterFunc; } return colFilter; @@ -56,11 +55,11 @@ export class SectionFilter extends Disposable { } /** - * Allows to override a single filter for a given fieldRef. Multiple calls to `setFilterOverride` will overwrite + * Allows to override a single filter for a given colRef. Multiple calls to `setFilterOverride` will overwrite * previously set values. */ - public setFilterOverride(fieldRef: number, colFilter: ColumnFilter) { - this._openFilterOverride.set(({fieldRef, colFilter})); + public setFilterOverride(colRef: number, colFilter: ColumnFilter) { + this._openFilterOverride.set(({colRef, colFilter})); colFilter.onDispose(() => { const override = this._openFilterOverride.get(); if (override && override.colFilter === colFilter) { @@ -81,8 +80,8 @@ export class SectionFilter extends Disposable { } /** - * Builds a filter function that combines the filter function of all the fields. You can use - * `getFilterFunc(field, colFilter)` to customize the filter func for each field. It calls + * Builds a filter function that combines the filter function of all the columns. You can use + * `getFilterFunc(column, colFilter)` to customize the filter func for each columns. It calls * `getFilterFunc` right away. Also, all the rows that were added with `addTemporaryRow()` bypass * the filter. */ @@ -96,16 +95,16 @@ export class SectionFilter extends Disposable { /** * Internal that helps build a filter function that combines the filter function of all - * fields. You can use `getFilterFunc(field, colFilter)` to customize the filter func for each - * field. It calls `getFilterFunc` right away + * columns. You can use `getFilterFunc(column, colFilter)` to customize the filter func for each + * column. It calls `getFilterFunc` right away. */ private _buildPlainFilterFunc(getFilterFunc: ColFilterCB, use: UseCB): RowFilterFunc { - const fields = use(use(this.viewFields).getObservable()); - const funcs: Array | null> = fields.map(f => { - const colFilter = buildColFilter(use(f.activeFilter), use(use(f.column).type)); - const filterFunc = getFilterFunc(f, colFilter); + const filters = use(this.viewSection.filters); + const funcs: Array | null> = filters.map(({filter, fieldOrColumn}) => { + const colFilter = buildColFilter(use(filter), use(use(fieldOrColumn.origCol).type)); + const filterFunc = getFilterFunc(fieldOrColumn, colFilter); - const getter = this._tableData.getRowPropFunc(f.colId.peek()); + const getter = this._tableData.getRowPropFunc(fieldOrColumn.colId.peek()); if (!filterFunc || !getter) { return null; } diff --git a/app/client/models/entities/ColumnRec.ts b/app/client/models/entities/ColumnRec.ts index fa453ca6..42b4fc80 100644 --- a/app/client/models/entities/ColumnRec.ts +++ b/app/client/models/entities/ColumnRec.ts @@ -3,6 +3,7 @@ import {DocModel, IRowModel, recordSet, refRecord, TableRec, ViewFieldRec} from import {jsonObservable, ObjObservable} from 'app/client/models/modelUtil'; import * as gristTypes from 'app/common/gristTypes'; import {getReferencedTableId} from 'app/common/gristTypes'; +import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter'; import * as ko from 'knockout'; // Represents a column in a user-defined table. @@ -35,6 +36,13 @@ export interface ColumnRec extends IRowModel<"_grist_Tables_column"> { // The column's display column _displayColModel: ko.Computed; + // Display col ref to use for the column, defaulting to the plain column itself. + displayColRef: ko.Computed; + + // The display column to use for the column, or the column itself when no displayCol is set. + displayColModel: ko.Computed; + visibleColModel: ko.Computed; + disableModifyBase: ko.Computed; // True if column config can't be modified (name, type, etc.) disableModify: ko.Computed; // True if column can't be modified or is being transformed. disableEditData: ko.Computed; // True to disable editing of the data in this column. @@ -46,6 +54,10 @@ export interface ColumnRec extends IRowModel<"_grist_Tables_column"> { // Helper which adds/removes/updates column's displayCol to match the formula. saveDisplayFormula(formula: string): Promise|undefined; + + // Helper for Reference/ReferenceList columns, which returns a formatter according + // to the visibleCol associated with column. Subscribes to observables if used within a computed. + createVisibleColFormatter(): BaseFormatter; } export function createColumnRec(this: ColumnRec, docModel: DocModel): void { @@ -84,6 +96,13 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void { } }; + // Display col ref to use for the column, defaulting to the plain column itself. + this.displayColRef = ko.pureComputed(() => this.displayCol() || this.origColRef()); + + // The display column to use for the column, or the column itself when no displayCol is set. + this.displayColModel = refRecord(docModel.columns, this.displayColRef); + this.visibleColModel = refRecord(docModel.columns, this.visibleCol); + this.disableModifyBase = ko.pureComputed(() => Boolean(this.summarySourceCol())); this.disableModify = ko.pureComputed(() => this.disableModifyBase() || this.isTransforming()); this.disableEditData = ko.pureComputed(() => Boolean(this.summarySourceCol())); @@ -95,4 +114,16 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void { const refTableId = getReferencedTableId(this.type() || ""); return refTableId ? docModel.allTables.all().find(t => t.tableId() === refTableId) || null : null; }); + + // Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol + // associated with this column. If no visible column available, return formatting for the column itself. + // Subscribes to observables if used within a computed. + // TODO: It would be better to replace this with a pureComputed whose value is a formatter. + this.createVisibleColFormatter = function() { + const vcol = this.visibleColModel(); + const documentSettings = docModel.docInfoRow.documentSettingsJson(); + return (vcol.getRowId() !== 0) ? + createFormatter(vcol.type(), vcol.widgetOptionsJson(), documentSettings) : + createFormatter(this.type(), this.widgetOptionsJson(), documentSettings); + }; } diff --git a/app/client/models/entities/FilterRec.ts b/app/client/models/entities/FilterRec.ts new file mode 100644 index 00000000..8dda93e1 --- /dev/null +++ b/app/client/models/entities/FilterRec.ts @@ -0,0 +1,22 @@ +import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel'; +import * as modelUtil from 'app/client/models/modelUtil'; +import * as ko from 'knockout'; + +// Represents a column filter for a view section. +export interface FilterRec extends IRowModel<"_grist_Filters"> { + viewSection: ko.Computed; + column: ko.Computed; + + // Observable for the parsed filter object. + activeFilter: modelUtil.CustomComputed; +} + +export function createFilterRec(this: FilterRec, docModel: DocModel): void { + this.viewSection = refRecord(docModel.viewSections, this.viewSectionRef); + this.column = refRecord(docModel.columns, this.colRef); + + // Observable for the active filter that's initialized from the value saved to the server. + this.activeFilter = modelUtil.customComputed({ + read: () => { const f = this.filter(); return f === 'null' ? '' : f; }, // To handle old empty filters. + }); +} diff --git a/app/client/models/entities/ViewFieldRec.ts b/app/client/models/entities/ViewFieldRec.ts index 8da6e225..eefd5939 100644 --- a/app/client/models/entities/ViewFieldRec.ts +++ b/app/client/models/entities/ViewFieldRec.ts @@ -7,7 +7,6 @@ import {DocumentSettings} from 'app/common/DocumentSettings'; import {isFullReferencingType} from 'app/common/gristTypes'; import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter'; import {createParser} from 'app/common/ValueParser'; -import {Computed, fromKo} from 'grainjs'; import * as ko from 'knockout'; // Represents a page entry in the tree of pages. @@ -65,12 +64,6 @@ export interface ViewFieldRec extends IRowModel<"_grist_Views_section_field"> { // Whether lines should wrap in a cell. wrapping: ko.Computed; - // Observable for the parsed filter object saved to the field. - activeFilter: modelUtil.CustomComputed; - - // Computed boolean that's true when there's a saved filter - isFiltered: Computed; - disableModify: ko.Computed; disableEditData: ko.Computed; @@ -232,14 +225,6 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void return this.widgetOptionsJson().wrap ?? (this.viewSection().parentKey() !== 'record'); }); - // Observable for the active filter that's initialized from the value saved to the server. - this.activeFilter = modelUtil.customComputed({ - read: () => { const f = this.filter(); return f === 'null' ? '' : f; }, // To handle old empty filters - save: (val) => this.filter.saveOnly(val), - }); - - this.isFiltered = Computed.create(this, fromKo(this.activeFilter), (_use, f) => f !== ''); - this.disableModify = ko.pureComputed(() => this.column().disableModify()); this.disableEditData = ko.pureComputed(() => this.column().disableEditData()); diff --git a/app/client/models/entities/ViewSectionRec.ts b/app/client/models/entities/ViewSectionRec.ts index 3d21b7c0..e816017f 100644 --- a/app/client/models/entities/ViewSectionRec.ts +++ b/app/client/models/entities/ViewSectionRec.ts @@ -1,5 +1,5 @@ import * as BaseView from 'app/client/components/BaseView'; -import { ColumnRec, TableRec, ViewFieldRec, ViewRec } from 'app/client/models/DocModel'; +import { ColumnRec, FilterRec, TableRec, ViewFieldRec, ViewRec } from 'app/client/models/DocModel'; import * as modelUtil from 'app/client/models/modelUtil'; import * as ko from 'knockout'; import { CursorPos, } from 'app/client/components/Cursor'; @@ -7,6 +7,7 @@ import { KoArray, } from 'app/client/lib/koArray'; import { DocModel, IRowModel, recordSet, refRecord, } from 'app/client/models/DocModel'; import { RowId, } from 'app/client/models/rowset'; import { getWidgetTypes, } from 'app/client/ui/widgetTypes'; +import { arrayRepeat, } from 'app/common/gutil'; import { Sort, } from 'app/common/SortSpec'; import { Computed, } from 'grainjs'; import defaults = require('lodash/defaults'); @@ -16,6 +17,9 @@ import defaults = require('lodash/defaults'); export interface ViewSectionRec extends IRowModel<"_grist_Views_section"> { viewFields: ko.Computed>; + // All table columns associated with this view section, excluding hidden helper columns. + columns: ko.Computed; + optionsObj: modelUtil.SaveableObjObservable; customDef: CustomViewSectionDef; @@ -33,13 +37,36 @@ export interface ViewSectionRec extends IRowModel<"_grist_Views_section"> { layoutSpecObj: modelUtil.ObjObservable; - // Helper metadata item which indicates whether any of the section's fields have unsaved + _savedFilters: ko.Computed>; + + /** + * Unsaved client-side filters, keyed by original col ref. Currently only wiped when unsaved filters + * are applied or reverted. + * + * If saved filters exist for a col ref, unsaved filters take priority and are applied instead. This + * prevents disruption when changes are made to saved filters for the same field/column, but there + * may be some cases where we'd want to reset _unsavedFilters on some indirect change to the document. + * + * NOTE: See `filters`, where `_unsavedFilters` is merged with `savedFilters`. + */ + _unsavedFilters: Map; + + /** + * Filter information for all fields/section in the section. + * + * Re-computed on changes to `savedFilters`, as well as any changes to `viewFields` or `columns`. Any + * unsaved filters saved in `_unsavedFilters` are applied on computation, taking priority over saved + * filters for the same field/column, if any exist. + */ + filters: ko.Computed; + + // Subset of `filters` containing non-blank active filters. + activeFilters: Computed; + + // Helper metadata item which indicates whether any of the section's fields/columns have unsaved // changes to their filters. (True indicates unsaved changes) filterSpecChanged: Computed; - // Array of fields with an active filter - filteredFields: Computed; - // Customizable version of the JSON-stringified sort spec. It may diverge from the saved one. activeSortJson: modelUtil.CustomComputed; @@ -96,14 +123,14 @@ export interface ViewSectionRec extends IRowModel<"_grist_Views_section"> { // We won't freeze all the columns on a grid, it will leave at least 1 column unfrozen. numFrozen: ko.Computed; - // Save all filters of fields in the section. + // Save all filters of fields/columns in the section. saveFilters(): Promise; - // Revert all filters of fields in the section. + // Revert all filters of fields/columns in the section. revertFilters(): void; - // Clear and save all filters of fields in the section. - clearFilters(): void; + // Apply `filter` to the field or column identified by `colRef`. + setFilter(colRef: number, filter: string): void; } export interface CustomViewSectionDef { @@ -129,10 +156,22 @@ export interface CustomViewSectionDef { sectionId: ko.Observable; } +// Information about filters for a field or hidden column. +export interface FilterInfo { + // The field or column associated with this filter info. + fieldOrColumn: ViewFieldRec|ColumnRec; + // Filter that applies to this field/column, if any. + filter: modelUtil.CustomComputed; + // True if `filter` has a non-blank value. + isFiltered: ko.PureComputed; +} export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel): void { this.viewFields = recordSet(this, docModel.viewFields, 'parentId', {sortBy: 'parentPos'}); + // All table columns associated with this view section, excluding any hidden helper columns. + this.columns = this.autoDispose(ko.pureComputed(() => this.table().columns().all().filter(c => !c.isHiddenCol()))); + const defaultOptions = { verticalGridlines: true, horizontalGridlines: true, @@ -180,28 +219,128 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel): this.layoutSpecObj = modelUtil.jsonObservable(this.layoutSpec); - // Helper metadata item which indicates whether any of the section's fields have unsaved + this._savedFilters = recordSet(this, docModel.filters, 'viewSectionRef'); + + /** + * Unsaved client-side filters, keyed by original col ref. Currently only wiped when unsaved filters + * are applied or reverted. + * + * If saved filters exist for a col ref, unsaved filters take priority and are applied instead. This + * prevents disruption when changes are made to saved filters for the same field/column, but there + * may be some cases where we'd want to reset _unsavedFilters on some indirect change to the document. + * + * NOTE: See `filters`, where `_unsavedFilters` is merged with `savedFilters`. + */ + this._unsavedFilters = new Map(); + + /** + * Filter information for all fields/section in the section. + * + * Re-computed on changes to `savedFilters`, as well as any changes to `viewFields` or `columns`. Any + * unsaved filters saved in `_unsavedFilters` are applied on computation, taking priority over saved + * filters for the same field/column, if any exist. + */ + this.filters = this.autoDispose(ko.computed(() => { + const savedFiltersByColRef = new Map(this._savedFilters().all().map(f => [f.colRef(), f])); + const viewFieldsByColRef = new Map(this.viewFields().all().map(f => [f.colRef(), f])); + + return this.columns().map(column => { + const savedFilter = savedFiltersByColRef.get(column.origColRef()); + const filter = modelUtil.customComputed({ + // Initialize with a saved filter, if one exists. Otherwise, use a blank filter. + read: () => { return savedFilter ? savedFilter.activeFilter() : ''; }, + }); + + // If an unsaved filter exists, overwrite `filter` with it. + const unsavedFilter = this._unsavedFilters.get(column.origColRef()); + if (unsavedFilter !== undefined) { filter(unsavedFilter); } + + return { + filter, + fieldOrColumn: viewFieldsByColRef.get(column.origColRef()) ?? column, + isFiltered: ko.pureComputed(() => filter() !== '') + }; + }); + })); + + // List of `filters` that have non-blank active filters. + this.activeFilters = Computed.create(this, use => use(this.filters).filter(col => use(col.isFiltered))); + + // Helper metadata item which indicates whether any of the section's fields/columns have unsaved // changes to their filters. (True indicates unsaved changes) - this.filterSpecChanged = Computed.create(this, use => - use(use(this.viewFields).getObservable()).some(field => !use(field.activeFilter.isSaved))); + this.filterSpecChanged = Computed.create(this, use => { + return use(this.filters).some(col => !use(col.filter.isSaved)); + }); - this.filteredFields = Computed.create(this, use => - use(use(this.viewFields).getObservable()).filter(field => use(field.isFiltered))); - - // Save all filters of fields in the section. + // Save all filters of fields/columns in the section. this.saveFilters = () => { return docModel.docData.bundleActions(`Save all filters in ${this.titleDef()}`, - async () => { await Promise.all(this.viewFields().all().map(field => field.activeFilter.save())); } + async () => { + const savedFiltersByColRef = new Map(this._savedFilters().all().map(f => [f.colRef(), f])); + const updatedFilters: [number, string][] = []; // Pairs of row ids and filters to update. + const removedFilterIds: number[] = []; // Row ids of filters to remove. + const newFilters: [number, string][] = []; // Pairs of column refs and filters to add. + + for (const c of this.filters()) { + // Skip saved filters (i.e. filters whose local values are unchanged from server). + if (c.filter.isSaved()) { continue; } + + const savedFilter = savedFiltersByColRef.get(c.fieldOrColumn.origCol().origColRef()); + if (!savedFilter) { + // Since no saved filter exists, we must add a new record to the filters table. + newFilters.push([c.fieldOrColumn.origCol().origColRef(), c.filter()]); + } else if (c.filter() === '') { + // Mark the saved filter for removal from the filters table. + removedFilterIds.push(savedFilter.id()); + } else { + // Mark the saved filter for update in the filters table. + updatedFilters.push([savedFilter.id(), c.filter()]); + } + } + + // Remove records of any deleted filters. + if (removedFilterIds.length > 0) { + await docModel.filters.sendTableAction(['BulkRemoveRecord', removedFilterIds]); + } + + // Update existing filter records with new filter values. + if (updatedFilters.length > 0) { + await docModel.filters.sendTableAction(['BulkUpdateRecord', + updatedFilters.map(([id]) => id), + {filter: updatedFilters.map(([, filter]) => filter)} + ]); + } + + // Add new filter records. + if (newFilters.length > 0) { + await docModel.filters.sendTableAction(['BulkAddRecord', + arrayRepeat(newFilters.length, null), + { + viewSectionRef: arrayRepeat(newFilters.length, this.id()), + colRef: newFilters.map(([colRef]) => colRef), + filter: newFilters.map(([, filter]) => filter), + } + ]); + } + + // Reset client filter state. + this.revertFilters(); + } ); }; - // Revert all filters of fields in the section. + // Revert all filters of fields/columns in the section. this.revertFilters = () => { - this.viewFields().all().forEach(field => { field.activeFilter.revert(); }); + this._unsavedFilters = new Map(); + this.filters().forEach(c => { c.filter.revert(); }); }; - // Reset all filters of fields in the section to their default (i.e. unset) values. - this.clearFilters = () => this.viewFields().all().forEach(field => field.activeFilter('')); + // Apply `filter` to the field or column identified by `colRef`. + this.setFilter = (colRef: number, filter: string) => { + this._unsavedFilters.set(colRef, filter); + const filterInfo = this.filters().find(c => c.fieldOrColumn.origCol().origColRef() === colRef); + filterInfo?.filter(filter); + }; // Customizable version of the JSON-stringified sort spec. It may diverge from the saved one. this.activeSortJson = modelUtil.customValue(this.sortColRefs); @@ -230,9 +369,7 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel): // Evaluates to an array of column models, which are not referenced by anything in viewFields. this.hiddenColumns = this.autoDispose(ko.pureComputed(() => { const included = new Set(this.viewFields().all().map((f) => f.column().origColRef())); - return this.table().columns().all().filter(function(col) { - return !included.has(col.getRowId()) && !col.isHiddenCol(); - }); + return this.columns().filter(c => !included.has(c.getRowId())); })); this.hasFocus = ko.pureComputed({ diff --git a/app/client/ui/ColumnFilterMenu.ts b/app/client/ui/ColumnFilterMenu.ts index fdd38c5f..608b3096 100644 --- a/app/client/ui/ColumnFilterMenu.ts +++ b/app/client/ui/ColumnFilterMenu.ts @@ -6,7 +6,8 @@ import {allInclusive, ColumnFilter} from 'app/client/models/ColumnFilter'; import {ColumnFilterMenuModel, IFilterCount} from 'app/client/models/ColumnFilterMenuModel'; -import {ViewFieldRec, ViewSectionRec} from 'app/client/models/DocModel'; +import {ColumnRec, ViewFieldRec, ViewSectionRec} from 'app/client/models/DocModel'; +import {FilterInfo} from 'app/client/models/entities/ViewSectionRec'; import {RowId, RowSource} from 'app/client/models/rowset'; import {ColumnFilterFunc, SectionFilter} from 'app/client/models/SectionFilter'; import {TableData} from 'app/client/models/TableData'; @@ -278,21 +279,22 @@ function formatUniqueCount(values: Array<[CellValue, IFilterCount]>) { /** * Returns content for the newly created columnFilterMenu; for use with setPopupToCreateDom(). */ -export function createFilterMenu(openCtl: IOpenController, sectionFilter: SectionFilter, field: ViewFieldRec, +export function createFilterMenu(openCtl: IOpenController, sectionFilter: SectionFilter, filterInfo: FilterInfo, rowSource: RowSource, tableData: TableData, onClose: () => void = noop) { // Go through all of our shown and hidden rows, and count them up by the values in this column. - const columnType = field.column().type.peek(); - const {keyMapFunc, labelMapFunc} = getMapFuncs(columnType, tableData, field); - const activeFilterBar = field.viewSection.peek().activeFilterBar; + const fieldOrColumn = filterInfo.fieldOrColumn; + const columnType = fieldOrColumn.origCol.peek().type.peek(); + const {keyMapFunc, labelMapFunc} = getMapFuncs(columnType, tableData, filterInfo.fieldOrColumn); + const activeFilterBar = sectionFilter.viewSection.activeFilterBar; - function getFilterFunc(f: ViewFieldRec, colFilter: ColumnFilterFunc|null) { - return f.getRowId() === field.getRowId() ? null : colFilter; + function getFilterFunc(col: ViewFieldRec|ColumnRec, colFilter: ColumnFilterFunc|null) { + return col.getRowId() === fieldOrColumn.getRowId() ? null : colFilter; } const filterFunc = Computed.create(null, use => sectionFilter.buildFilterFunc(getFilterFunc, use)); openCtl.autoDispose(filterFunc); - const columnFilter = ColumnFilter.create(openCtl, field.activeFilter.peek(), columnType); - sectionFilter.setFilterOverride(field.getRowId(), columnFilter); // Will be removed on menu disposal + const columnFilter = ColumnFilter.create(openCtl, filterInfo.filter.peek(), columnType); + sectionFilter.setFilterOverride(fieldOrColumn.getRowId(), columnFilter); // Will be removed on menu disposal const [allRows, hiddenRows] = partition(Array.from(rowSource.getAllRows()), filterFunc.get()); const valueCounts: Map = new Map(); @@ -310,12 +312,15 @@ export function createFilterMenu(openCtl: IOpenController, sectionFilter: Sectio doSave: (reset: boolean = false) => { const spec = columnFilter.makeFilterJson(); // If filter is moot and filter bar is hidden, let's remove the filter. - field.activeFilter((spec === allInclusive && !activeFilterBar.peek()) ? '' : spec); + sectionFilter.viewSection.setFilter( + fieldOrColumn.origCol().origColRef(), + spec === allInclusive && !activeFilterBar.peek() ? '' : spec + ); if (reset) { sectionFilter.resetTemporaryRows(); } }, - renderValue: getRenderFunc(columnType, field), + renderValue: getRenderFunc(columnType, fieldOrColumn), }); } @@ -330,10 +335,10 @@ export function createFilterMenu(openCtl: IOpenController, sectionFilter: Sectio * Used by ColumnFilterMenu to compute counts of unique cell * values and display them with an appropriate label. */ -function getMapFuncs(columnType: string, tableData: TableData, field: ViewFieldRec) { - const keyMapFunc = tableData.getRowPropFunc(field.column().colId())!; - const labelGetter = tableData.getRowPropFunc(field.displayColModel().colId())!; - const formatter = field.createVisibleColFormatter(); +function getMapFuncs(columnType: string, tableData: TableData, fieldOrColumn: ViewFieldRec|ColumnRec) { + const keyMapFunc = tableData.getRowPropFunc(fieldOrColumn.colId())!; + const labelGetter = tableData.getRowPropFunc(fieldOrColumn.displayColModel().colId())!; + const formatter = fieldOrColumn.createVisibleColFormatter(); let labelMapFunc: (rowId: number) => string | string[]; if (isRefListType(columnType)) { @@ -357,9 +362,9 @@ function getMapFuncs(columnType: string, tableData: TableData, field: ViewFieldR * column types by rendering their values as colored tokens instead of * text. */ -function getRenderFunc(columnType: string, field: ViewFieldRec) { +function getRenderFunc(columnType: string, fieldOrColumn: ViewFieldRec|ColumnRec) { if (['Choice', 'ChoiceList'].includes(columnType)) { - const options = field.column().widgetOptionsJson.peek(); + const options = fieldOrColumn.widgetOptionsJson.peek(); const choiceSet: Set = new Set(options.choices || []); const choiceOptions: ChoiceOptions = options.choiceOptions || {}; @@ -460,13 +465,14 @@ interface IColumnFilterMenuOptions extends IPopupOptions { } // Helper to attach the column filter menu. -export function attachColumnFilterMenu(viewSection: ViewSectionRec, field: ViewFieldRec, +export function attachColumnFilterMenu(viewSection: ViewSectionRec, filterInfo: FilterInfo, popupOptions: IColumnFilterMenuOptions): DomElementMethod { const options = {...defaultPopupOptions, ...popupOptions}; return (elem) => { const instance = viewSection.viewInstance(); if (instance && instance.createFilterMenu) { // Should be set if using BaseView - setPopupToCreateDom(elem, ctl => instance.createFilterMenu(ctl, field, popupOptions.onCloseContent), options); + setPopupToCreateDom(elem, ctl => + instance.createFilterMenu(ctl, filterInfo, popupOptions.onCloseContent), options); } }; } diff --git a/app/client/ui/FilterBar.ts b/app/client/ui/FilterBar.ts index a459de2b..6b229e7d 100644 --- a/app/client/ui/FilterBar.ts +++ b/app/client/ui/FilterBar.ts @@ -1,5 +1,6 @@ import { allInclusive } from "app/client/models/ColumnFilter"; -import { ViewFieldRec, ViewSectionRec } from "app/client/models/DocModel"; +import { ColumnRec, ViewFieldRec, ViewSectionRec } from "app/client/models/DocModel"; +import { FilterInfo } from "app/client/models/entities/ViewSectionRec"; import { attachColumnFilterMenu } from "app/client/ui/ColumnFilterMenu"; import { cssButton, cssButtonGroup } from "app/client/ui2018/buttons"; import { colors, testId } from "app/client/ui2018/cssVars"; @@ -9,46 +10,46 @@ import { dom, IDisposableOwner, IDomArgs, styled } from "grainjs"; import { IMenuOptions, PopupControl } from "popweasel"; export function filterBar(_owner: IDisposableOwner, viewSection: ViewSectionRec) { - const popupControls = new WeakMap(); + const popupControls = new WeakMap(); return cssFilterBar( testId('filter-bar'), - dom.forEach(viewSection.filteredFields, (field) => makeFilterField(viewSection, field, popupControls)), + dom.forEach(viewSection.activeFilters, (filterInfo) => makeFilterField(viewSection, filterInfo, popupControls)), makePlusButton(viewSection, popupControls), ); } -function makeFilterField(viewSection: ViewSectionRec, field: ViewFieldRec, - popupControls: WeakMap) { +function makeFilterField(viewSection: ViewSectionRec, filterInfo: FilterInfo, + popupControls: WeakMap) { return cssFilterBarItem( testId('filter-field'), primaryButton( testId('btn'), cssIcon('FilterSimple'), - cssMenuTextLabel(dom.text(field.label)), - cssBtn.cls('-grayed', field.activeFilter.isSaved), - attachColumnFilterMenu(viewSection, field, { + cssMenuTextLabel(dom.text(filterInfo.fieldOrColumn.label)), + cssBtn.cls('-grayed', filterInfo.filter.isSaved), + attachColumnFilterMenu(viewSection, filterInfo, { placement: 'bottom-start', attach: 'body', - trigger: ['click', (_el, popupControl) => popupControls.set(field, popupControl)] + trigger: ['click', (_el, popupControl) => popupControls.set(filterInfo.fieldOrColumn.origCol(), popupControl)] }), ), deleteButton( testId('delete'), cssIcon('CrossSmall'), - cssBtn.cls('-grayed', field.activeFilter.isSaved), - dom.on('click', () => field.activeFilter('')), + cssBtn.cls('-grayed', filterInfo.filter.isSaved), + dom.on('click', () => viewSection.setFilter(filterInfo.fieldOrColumn.origCol().origColRef(), '')), ) ); } -export function addFilterMenu(fields: ViewFieldRec[], popupControls: WeakMap, - options?: IMenuOptions) { +export function addFilterMenu(filters: FilterInfo[], viewSection: ViewSectionRec, + popupControls: WeakMap, options?: IMenuOptions) { return ( menu((ctl) => [ - ...fields.map((f) => ( + ...filters.map((filterInfo) => ( menuItemAsync( - () => turnOnAndOpenFilter(f, popupControls), - f.label.peek(), - dom.cls('disabled', f.isFiltered), + () => turnOnAndOpenFilter(filterInfo.fieldOrColumn, viewSection, popupControls), + filterInfo.fieldOrColumn.label.peek(), + dom.cls('disabled', filterInfo.isFiltered), testId('add-filter-item'), ) )), @@ -62,19 +63,20 @@ export function addFilterMenu(fields: ViewFieldRec[], popupControls: WeakMap) { - f.activeFilter(allInclusive); - popupControls.get(f)?.open(); +function turnOnAndOpenFilter(fieldOrColumn: ViewFieldRec|ColumnRec, viewSection: ViewSectionRec, + popupControls: WeakMap) { + viewSection.setFilter(fieldOrColumn.origCol().origColRef(), allInclusive); + popupControls.get(fieldOrColumn.origCol())?.open(); } -function makePlusButton(viewSectionRec: ViewSectionRec, popupControls: WeakMap) { +function makePlusButton(viewSectionRec: ViewSectionRec, popupControls: WeakMap) { return dom.domComputed((use) => { - const fields = use(use(viewSectionRec.viewFields).getObservable()); - const anyFilter = fields.find((f) => use(f.isFiltered)); + const filters = use(viewSectionRec.filters); + const anyFilter = use(viewSectionRec.activeFilters).length > 0; return cssPlusButton( cssBtn.cls('-grayed'), cssIcon('Plus'), - addFilterMenu(fields, popupControls), + addFilterMenu(filters, viewSectionRec, popupControls), anyFilter ? null : cssPlusLabel('Add Filter'), testId('add-filter-btn') ); diff --git a/app/client/ui/ViewSectionMenu.ts b/app/client/ui/ViewSectionMenu.ts index a8d2cb87..32b0b1ce 100644 --- a/app/client/ui/ViewSectionMenu.ts +++ b/app/client/ui/ViewSectionMenu.ts @@ -1,5 +1,6 @@ import { reportError } from 'app/client/models/AppModel'; -import { ColumnRec, DocModel, ViewFieldRec, ViewRec, ViewSectionRec } from 'app/client/models/DocModel'; +import { ColumnRec, DocModel, ViewRec, ViewSectionRec } from 'app/client/models/DocModel'; +import { FilterInfo } from 'app/client/models/entities/ViewSectionRec'; import { CustomComputed } from 'app/client/models/modelUtil'; import { attachColumnFilterMenu } from 'app/client/ui/ColumnFilterMenu'; import { addFilterMenu } from 'app/client/ui/FilterBar'; @@ -35,8 +36,8 @@ function doRevert(viewSection: ViewSectionRec) { export function viewSectionMenu(owner: IDisposableOwner, docModel: DocModel, viewSection: ViewSectionRec, viewModel: ViewRec, isReadonly: Observable) { - const popupControls = new WeakMap(); - const anyFilter = Computed.create(owner, (use) => Boolean(use(viewSection.filteredFields).length)); + const popupControls = new WeakMap(); + const anyFilter = Computed.create(owner, (use) => Boolean(use(viewSection.activeFilters).length)); const displaySaveObs: Computed = Computed.create(owner, (use) => ( use(viewSection.filterSpecChanged) @@ -65,8 +66,8 @@ export function viewSectionMenu(owner: IDisposableOwner, docModel: DocModel, vie return makeSortPanel(viewSection, use(viewSection.activeSortSpec), (row: number) => docModel.columns.getRowModel(row)); }), - dom.domComputed(viewSection.filteredFields, fields => - makeFilterPanel(viewSection, fields, popupControls, () => ctl.close())), + dom.domComputed(viewSection.activeFilters, filters => + makeFilterPanel(viewSection, filters, popupControls, () => ctl.close())), makeAddFilterButton(viewSection, popupControls), makeFilterBarToggle(viewSection.activeFilterBar), dom.domComputed(displaySaveObs, displaySave => [ @@ -139,14 +140,13 @@ function makeSortPanel(section: ViewSectionRec, sortSpec: Sort.SortSpec, getColu ]; } -export function makeAddFilterButton(viewSectionRec: ViewSectionRec, - popupControls: WeakMap) { +export function makeAddFilterButton(viewSectionRec: ViewSectionRec, popupControls: WeakMap) { return dom.domComputed((use) => { - const fields = use(use(viewSectionRec.viewFields).getObservable()); + const filters = use(viewSectionRec.filters); return cssMenuText( cssMenuIconWrapper( cssIcon('Plus'), - addFilterMenu(fields, popupControls, { + addFilterMenu(filters, viewSectionRec, popupControls, { placement: 'bottom-end', // Attach content to triggerElem's parent, which is needed to prevent view section menu to // close when clicking an item of the add filter menu. @@ -179,32 +179,37 @@ export function makeFilterBarToggle(activeFilterBar: CustomComputed) { } -function makeFilterPanel(section: ViewSectionRec, filteredFields: ViewFieldRec[], - popupControls: WeakMap, +function makeFilterPanel(section: ViewSectionRec, activeFilters: FilterInfo[], + popupControls: WeakMap, onCloseContent: () => void) { - const fields = filteredFields.map(field => { - const fieldChanged = Computed.create(null, fromKo(field.activeFilter.isSaved), (_use, isSaved) => !isSaved); + const filters = activeFilters.map(filterInfo => { + const filterChanged = Computed.create(null, fromKo(filterInfo.filter.isSaved), (_use, isSaved) => !isSaved); return cssMenuText( - dom.autoDispose(fieldChanged), cssMenuIconWrapper( - cssMenuIconWrapper.cls('-changed', fieldChanged), + cssMenuIconWrapper.cls('-changed', filterChanged), cssIcon('FilterSimple'), - attachColumnFilterMenu(section, field, { + attachColumnFilterMenu(section, filterInfo, { placement: 'bottom-end', - trigger: ['click', (_el, popupControl) => popupControls.set(field, popupControl)], + trigger: [ + 'click', + (_el, popupControl) => popupControls.set(filterInfo.fieldOrColumn.origCol(), popupControl) + ], onCloseContent, }), testId('filter-icon'), ), - cssMenuTextLabel(field.label()), - cssMenuIconWrapper(cssIcon('Remove', testId('btn-remove-filter')), dom.on('click', () => field.activeFilter(''))), + cssMenuTextLabel(filterInfo.fieldOrColumn.label()), + cssMenuIconWrapper(cssIcon('Remove', + dom.on('click', () => section.setFilter(filterInfo.fieldOrColumn.origCol().origColRef(), ''))), + testId('btn-remove-filter') + ), testId('filter-col') ); }); return [ cssMenuInfoHeader('Filtered by', {style: 'margin-top: 4px'}, testId('heading-filtered')), - filteredFields.length > 0 ? fields : cssGrayedMenuText('(Not filtered)') + activeFilters.length > 0 ? filters : cssGrayedMenuText('(Not filtered)') ]; } diff --git a/app/common/schema.ts b/app/common/schema.ts index 66669e4f..1db1e3cf 100644 --- a/app/common/schema.ts +++ b/app/common/schema.ts @@ -4,7 +4,7 @@ import { GristObjCode } from "app/plugin/GristData"; // tslint:disable:object-literal-key-quotes -export const SCHEMA_VERSION = 24; +export const SCHEMA_VERSION = 25; export const schema = { @@ -185,6 +185,12 @@ export const schema = { child : "Ref:_grist_ACLPrincipals", }, + "_grist_Filters": { + viewSectionRef : "Ref:_grist_Views_section", + colRef : "Ref:_grist_Tables_column", + filter : "Text", + }, + }; export interface SchemaTypes { @@ -366,4 +372,10 @@ export interface SchemaTypes { child: number; }; + "_grist_Filters": { + viewSectionRef: number; + colRef: number; + filter: string; + }; + } diff --git a/app/server/lib/Export.ts b/app/server/lib/Export.ts index 1d872f1a..f4313cf4 100644 --- a/app/server/lib/Export.ts +++ b/app/server/lib/Export.ts @@ -217,33 +217,39 @@ export async function exportSection( .filterRecords({ parentId: table.id }) as GristTablesColumn[]; const viewSectionFields = safeTable(docData, '_grist_Views_section_field'); const fields = viewSectionFields.filterRecords({ parentId: viewSection.id }) as GristViewsSectionField[]; + const savedFilters = safeTable(docData, '_grist_Filters') + .filterRecords({ viewSectionRef: viewSection.id }) as GristFilter[]; const tableColsById = _.indexBy(columns, 'id'); + const fieldsByColRef = _.indexBy(fields, 'colRef'); + const savedFiltersByColRef = _.indexBy(savedFilters, 'colRef'); + const unsavedFiltersByColRef = _.indexBy(filters ?? [], 'colRef'); // Produce a column description matching what user will see / expect to export - const viewify = (col: GristTablesColumn, field: GristViewsSectionField) => { - field = field || {}; - const displayCol = tableColsById[field.displayCol || col.displayCol || col.id]; + const viewify = (col: GristTablesColumn, field?: GristViewsSectionField) => { + const displayCol = tableColsById[field?.displayCol || col.displayCol || col.id]; const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {}); - const fieldWidgetOptions = gutil.safeJsonParse(field.widgetOptions, {}); - const filterString = (filters || []).find(x => x.colRef === field.colRef)?.filter || field.filter; + const fieldWidgetOptions = field ? gutil.safeJsonParse(field.widgetOptions, {}) : {}; + const filterString = unsavedFiltersByColRef[col.id]?.filter || savedFiltersByColRef[col.id]?.filter; const filterFunc = buildColFilter(filterString, col.type); return { + filterFunc, id: displayCol.id, colId: displayCol.colId, label: col.label, type: col.type, parentPos: col.parentPos, - filterFunc, - widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions) + widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions), }; }; - const viewColumns = _.sortBy(fields, 'parentPos').map( - (field) => viewify(tableColsById[field.colRef], field)); + const tableColumns = columns + .filter(column => !gristTypes.isHiddenCol(column.colId)) + .map(column => viewify(column, fieldsByColRef[column.id])); + const viewColumns = _.sortBy(fields, 'parentPos') + .map((field) => viewify(tableColsById[field.colRef], field)); // The columns named in sort order need to now become display columns sortSpec = sortSpec || gutil.safeJsonParse(viewSection.sortColRefs, []); - const fieldsByColRef = _.indexBy(fields, 'colRef'); sortSpec = sortSpec!.map((colSpec) => { const colRef = Sort.getColRef(colSpec); const col = tableColsById[colRef]; @@ -264,10 +270,10 @@ export async function exportSection( sorter.updateSpec(sortSpec); rowIds.sort((a, b) => sorter.compare(a, b)); // create cell accessors - const access = viewColumns.map(col => getters.getColGetter(col.id)!); + const tableAccess = tableColumns.map(col => getters.getColGetter(col.id)!); // create row filter based on all columns filter - const rowFilter = viewColumns - .map((col, c) => buildRowFilter(access[c], col.filterFunc)) + const rowFilter = tableColumns + .map((col, c) => buildRowFilter(tableAccess[c], col.filterFunc)) .reduce((prevFilter, curFilter) => (id) => prevFilter(id) && curFilter(id), () => true); // filter rows numbers rowIds = rowIds.filter(rowFilter); @@ -276,11 +282,11 @@ export async function exportSection( const docSettings = gutil.safeJsonParse(docInfo.documentSettings, {}); return { + rowIds, + docSettings, tableName: table.tableId, docName: activeDoc.docName, - rowIds, - access, - docSettings, + access: viewColumns.map(col => getters.getColGetter(col.id)!), columns: viewColumns }; } @@ -294,6 +300,7 @@ type GristTables = RowModel<'_grist_Tables'> type GristViewsSectionField = RowModel<'_grist_Views_section_field'> type GristTablesColumn = RowModel<'_grist_Tables_column'> type GristView = RowModel<'_grist_Views'> +type GristFilter = RowModel<'_grist_Filters'> type DocInfo = RowModel<'_grist_DocInfo'> // Type for filters passed from the client diff --git a/app/server/lib/initialDocSql.ts b/app/server/lib/initialDocSql.ts index b24e4563..9ba79a5f 100644 --- a/app/server/lib/initialDocSql.ts +++ b/app/server/lib/initialDocSql.ts @@ -6,7 +6,7 @@ export const GRIST_DOC_SQL = ` PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT ''); -INSERT INTO _grist_DocInfo VALUES(1,'','','',24,'UTC','{"locale": "en-US"}'); +INSERT INTO _grist_DocInfo VALUES(1,'','','',25,'UTC','{"locale": "en-US"}'); CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0); CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL); CREATE TABLE IF NOT EXISTS "_grist_Imports" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "origFileName" TEXT DEFAULT '', "parseFormula" TEXT DEFAULT '', "delimiter" TEXT DEFAULT '', "doublequote" BOOLEAN DEFAULT 0, "escapechar" TEXT DEFAULT '', "quotechar" TEXT DEFAULT '', "skipinitialspace" BOOLEAN DEFAULT 0, "encoding" TEXT DEFAULT '', "hasHeaders" BOOLEAN DEFAULT 0); @@ -33,6 +33,7 @@ INSERT INTO _grist_ACLPrincipals VALUES(2,'group','','','Admins',''); INSERT INTO _grist_ACLPrincipals VALUES(3,'group','','','Editors',''); INSERT INTO _grist_ACLPrincipals VALUES(4,'group','','','Viewers',''); CREATE TABLE IF NOT EXISTS "_grist_ACLMemberships" (id INTEGER PRIMARY KEY, "parent" INTEGER DEFAULT 0, "child" INTEGER DEFAULT 0); +CREATE TABLE IF NOT EXISTS "_grist_Filters" (id INTEGER PRIMARY KEY, "viewSectionRef" INTEGER DEFAULT 0, "colRef" INTEGER DEFAULT 0, "filter" TEXT DEFAULT ''); COMMIT; `; @@ -40,7 +41,7 @@ export const GRIST_DOC_WITH_TABLE1_SQL = ` PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT ''); -INSERT INTO _grist_DocInfo VALUES(1,'','','',24,'UTC','{"locale": "en-US"}'); +INSERT INTO _grist_DocInfo VALUES(1,'','','',25,'UTC','{"locale": "en-US"}'); CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0); INSERT INTO _grist_Tables VALUES(1,'Table1',1,0,0); CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL); @@ -79,6 +80,7 @@ INSERT INTO _grist_ACLPrincipals VALUES(2,'group','','','Admins',''); INSERT INTO _grist_ACLPrincipals VALUES(3,'group','','','Editors',''); INSERT INTO _grist_ACLPrincipals VALUES(4,'group','','','Viewers',''); CREATE TABLE IF NOT EXISTS "_grist_ACLMemberships" (id INTEGER PRIMARY KEY, "parent" INTEGER DEFAULT 0, "child" INTEGER DEFAULT 0); +CREATE TABLE IF NOT EXISTS "_grist_Filters" (id INTEGER PRIMARY KEY, "viewSectionRef" INTEGER DEFAULT 0, "colRef" INTEGER DEFAULT 0, "filter" TEXT DEFAULT ''); CREATE TABLE IF NOT EXISTS "Table1" (id INTEGER PRIMARY KEY, "manualSort" NUMERIC DEFAULT 1e999, "A" BLOB DEFAULT NULL, "B" BLOB DEFAULT NULL, "C" BLOB DEFAULT NULL); COMMIT; `; diff --git a/sandbox/grist/docmodel.py b/sandbox/grist/docmodel.py index 22b747a1..f32f2c69 100644 --- a/sandbox/grist/docmodel.py +++ b/sandbox/grist/docmodel.py @@ -107,6 +107,11 @@ class MetaTableExtras(object): class _grist_Views_section(object): fields = _record_set('_grist_Views_section_field', 'parentId', sort_by='parentPos') + class _grist_Filters(object): + def setAutoRemove(rec, table): + """Marks the filter for removal if its column no longer exists.""" + table.docmodel.setAutoRemove(rec, not rec.colRef) + def enhance_model(model_class): """ @@ -159,6 +164,7 @@ class DocModel(object): self.pages = self._prep_table("_grist_Pages") self.aclResources = self._prep_table("_grist_ACLResources") self.aclRules = self._prep_table("_grist_ACLRules") + self.filters = self._prep_table("_grist_Filters") def _prep_table(self, name): """ diff --git a/sandbox/grist/migrations.py b/sandbox/grist/migrations.py index fbbef779..daff4f80 100644 --- a/sandbox/grist/migrations.py +++ b/sandbox/grist/migrations.py @@ -807,3 +807,36 @@ def migration24(tdset): schema.make_column("actions", "Text"), # JSON ]), ]) + +@migration(schema_version=25) +def migration25(tdset): + """ + Add _grist_Filters table and populate based on existing filters stored + in _grist_Views_section_field. + + From this version on, filter in _grist_Views_section_field is deprecated. + """ + doc_actions = [ + actions.AddTable('_grist_Filters', [ + schema.make_column("viewSectionRef", "Ref:_grist_Views_section"), + schema.make_column("colRef", "Ref:_grist_Tables_column"), + schema.make_column("filter", "Text"), + ]) + ] + + # Move existing field filters to _grist_Filters. + fields = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Views_section_field'])) + col_info = { 'filter': [], 'colRef': [], 'viewSectionRef': [] } + for f in fields: + if not f.filter: + continue + + col_info['filter'].append(f.filter) + col_info['colRef'].append(f.colRef) + col_info['viewSectionRef'].append(f.parentId) + + num_filters = len(col_info['filter']) + if num_filters > 0: + doc_actions.append(actions.BulkAddRecord('_grist_Filters', [None] * num_filters, col_info)) + + return tdset.apply_doc_actions(doc_actions) diff --git a/sandbox/grist/schema.py b/sandbox/grist/schema.py index e904490c..f41b94aa 100644 --- a/sandbox/grist/schema.py +++ b/sandbox/grist/schema.py @@ -15,7 +15,7 @@ import six import actions -SCHEMA_VERSION = 24 +SCHEMA_VERSION = 25 def make_column(col_id, col_type, formula='', isFormula=False): return { @@ -202,11 +202,8 @@ def schema_create_actions(): make_column("displayCol", "Ref:_grist_Tables_column"), # For Ref cols only, may override the column to be displayed fromin the pointed-to table. make_column("visibleCol", "Ref:_grist_Tables_column"), - # JSON string describing the default filter as map from either an `included` or an - # `excluded` string to an array of column values: - # Ex1: { included: ['foo', 'bar'] } - # Ex2: { excluded: ['apple', 'orange'] } - make_column("filter", "Text") + # DEPRECATED: replaced with _grist_Filters in version 25. Do not remove or reuse. + make_column("filter", "Text"), ]), # The code for all of the validation rules available to a Grist document @@ -301,6 +298,16 @@ def schema_create_actions(): make_column('parent', 'Ref:_grist_ACLPrincipals'), make_column('child', 'Ref:_grist_ACLPrincipals'), ]), + + actions.AddTable('_grist_Filters', [ + make_column("viewSectionRef", "Ref:_grist_Views_section"), + make_column("colRef", "Ref:_grist_Tables_column"), + # JSON string describing the default filter as map from either an `included` or an + # `excluded` string to an array of column values: + # Ex1: { included: ['foo', 'bar'] } + # Ex2: { excluded: ['apple', 'orange'] } + make_column("filter", "Text") + ]), ]