From 116fb15edab69abd19eb65eb475cfccc10d6bd7c Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Mon, 6 Dec 2021 14:07:52 +0200 Subject: [PATCH] (core) Move most of the reference parsing code into common so that the server can use it Summary: Refactoring in preparation for parsing strings from the API. The plan is that the API code will only need to do a server-side version of the code in ViewFieldRec.valueParser (minus ReferenceUtils) which is quite minimal. Test Plan: Nothing extra here, I don't think it's needed. This stuff will get tested more in a future diff which changes the API. Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D3164 --- app/client/lib/ReferenceUtils.ts | 79 +------------- app/client/models/TableData.ts | 29 ----- app/client/models/entities/ViewFieldRec.ts | 34 ++---- app/common/TableData.ts | 5 +- app/common/ValueParser.ts | 120 ++++++++++++++++++++- 5 files changed, 130 insertions(+), 137 deletions(-) diff --git a/app/client/lib/ReferenceUtils.ts b/app/client/lib/ReferenceUtils.ts index ed6e59a8..e811b701 100644 --- a/app/client/lib/ReferenceUtils.ts +++ b/app/client/lib/ReferenceUtils.ts @@ -1,10 +1,9 @@ -import { DocData } from 'app/client/models/DocData'; +import {DocData} from 'app/client/models/DocData'; import {ColumnRec} from 'app/client/models/entities/ColumnRec'; import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec'; -import {SearchFunc, TableData} from 'app/client/models/TableData'; +import {TableData} from 'app/client/models/TableData'; import {getReferencedTableId, isRefListType} from 'app/common/gristTypes'; import {BaseFormatter} from 'app/common/ValueFormatter'; -import isEqual = require('lodash/isEqual'); /** * Utilities for common operations involving Ref[List] fields. @@ -40,80 +39,6 @@ export class ReferenceUtils { this.isRefList = isRefListType(colType); } - public parseReference( - raw: string, value: unknown - ): number | string | ['l', unknown, {raw?: string, column: string}] { - if (!value || !raw) { - return 0; // default value for a reference column - } - - if (this.visibleColId === 'id') { - const n = Number(value); - if (Number.isInteger(n)) { - value = n; - } else { - return raw; - } - } - - if (!this.tableData.isLoaded) { - const options: {column: string, raw?: string} = {column: this.visibleColId}; - if (value !== raw) { - options.raw = raw; - } - return ['l', value, options]; - } - - const searchFunc: SearchFunc = (v: any) => isEqual(v, value); - const matches = this.tableData.columnSearch(this.visibleColId, searchFunc, 1); - if (matches.length > 0) { - return matches[0]; - } else { - // There's no matching value in the visible column, i.e. this is not a valid reference. - // We need to return a string which will become AltText. - return raw; - } - } - - public parseReferenceList( - raw: string, values: unknown[] - ): ['L', ...number[]] | null | string | ['l', unknown[], {raw?: string, column: string}] { - if (!values.length || !raw) { - return null; // default value for a reference list column - } - - if (this.visibleColId === 'id') { - const numbers = values.map(Number); - if (numbers.every(Number.isInteger)) { - values = numbers; - } else { - return raw; - } - } - - if (!this.tableData.isLoaded) { - const options: {column: string, raw?: string} = {column: this.visibleColId}; - if (!(values.length === 1 && values[0] === raw)) { - options.raw = raw; - } - return ['l', values, options]; - } - - const rowIds: number[] = []; - for (const value of values) { - const searchFunc: SearchFunc = (v: any) => isEqual(v, value); - const matches = this.tableData.columnSearch(this.visibleColId, searchFunc, 1); - if (matches.length > 0) { - rowIds.push(matches[0]); - } else { - // There's no matching value in the visible column, i.e. this is not a valid reference. - // We need to return a string which will become AltText. - return raw; - } - } - return ['L', ...rowIds]; - } - public idToText(value: unknown) { if (typeof value === 'number') { return this.formatter.formatAny(this.tableData.getValue(value, this.visibleColId)); diff --git a/app/client/models/TableData.ts b/app/client/models/TableData.ts index 35c582f4..d0fd0ea2 100644 --- a/app/client/models/TableData.ts +++ b/app/client/models/TableData.ts @@ -10,8 +10,6 @@ import { countIf } from 'app/common/gutil'; import { TableData as BaseTableData, ColTypeMap } from 'app/common/TableData'; import { Emitter } from 'grainjs'; -export type SearchFunc = (value: any) => boolean; - /** * TableData class to maintain a single table's data. */ @@ -60,33 +58,6 @@ export class TableData extends BaseTableData { this.dataLoadedEmitter.emit(rowIds, []); } - /** - * Given a colId and a search function, returns a list of matching row IDs, optionally limiting their number. - * @param {String} colId: identifies the column to search. - * @param {Function} searchFunc: A function which, given a column value, returns whether to include it. - * @param [Number] optMaxResults: if given, limit the number of returned results to this. - * @returns Array[Number] array of row IDs. - */ - public columnSearch(colId: string, searchFunc: SearchFunc, optMaxResults?: number) { - const maxResults = optMaxResults || Number.POSITIVE_INFINITY; - - const rowIds = this.getRowIds(); - const valColumn = this.getColValues(colId); - const ret = []; - if (!valColumn) { - // tslint:disable-next-line:no-console - console.warn(`TableData.columnSearch called on invalid column ${this.tableId}.${colId}`); - } else { - for (let i = 0; i < rowIds.length && ret.length < maxResults; i++) { - const value = valColumn[i]; - if (value && searchFunc(value)) { - ret.push(rowIds[i]); - } - } - } - return ret; - } - /** * Counts and returns the number of error values in the given column. The count is cached to * keep it faster for large tables, and the cache is cleared as needed on changes to the table. diff --git a/app/client/models/entities/ViewFieldRec.ts b/app/client/models/entities/ViewFieldRec.ts index eefd5939..0fc9de17 100644 --- a/app/client/models/entities/ViewFieldRec.ts +++ b/app/client/models/entities/ViewFieldRec.ts @@ -1,10 +1,8 @@ -import {ReferenceUtils} from 'app/client/lib/ReferenceUtils'; import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel'; import * as modelUtil from 'app/client/models/modelUtil'; import * as UserType from 'app/client/widgets/UserType'; -import {csvDecodeRow} from 'app/common/csvFormat'; import {DocumentSettings} from 'app/common/DocumentSettings'; -import {isFullReferencingType} from 'app/common/gristTypes'; +import {getReferencedTableId, isFullReferencingType} from 'app/common/gristTypes'; import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter'; import {createParser} from 'app/common/ValueParser'; import * as ko from 'knockout'; @@ -180,31 +178,15 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void const docSettings = this.documentSettings(); const type = this.column().type(); - if (!isFullReferencingType(type)) { - return createParser(type, this.widgetOptionsJson(), docSettings); - } else { + const widgetOpts = this.widgetOptionsJson(); + if (isFullReferencingType(type)) { const vcol = this.visibleColModel(); - const vcolParser = createParser(vcol.type(), vcol.widgetOptionsJson(), docSettings); - const refUtils = new ReferenceUtils(this, docModel.docData); // uses several more observables immediately - if (!refUtils.isRefList) { - return (s: string) => refUtils.parseReference(s, vcolParser(s)); - } else { - return (s: string) => { - let values: any[] | null; - try { - values = JSON.parse(s); - } catch { - values = null; - } - if (!Array.isArray(values)) { - // csvDecodeRow should never raise an exception - values = csvDecodeRow(s); - } - values = values.map(v => typeof v === "string" ? vcolParser(v) : v); - return refUtils.parseReferenceList(s, values); - }; - } + widgetOpts.visibleColId = vcol.colId() || 'id'; + widgetOpts.visibleColType = vcol.type(); + widgetOpts.visibleColWidgetOpts = vcol.widgetOptionsJson(); + widgetOpts.tableData = docModel.docData.getTable(getReferencedTableId(type)!); } + return createParser(type, widgetOpts, docSettings); }); // The widgetOptions to read and write: either the column's or the field's own. diff --git a/app/common/TableData.ts b/app/common/TableData.ts index 131c1303..d9dcd86c 100644 --- a/app/common/TableData.ts +++ b/app/common/TableData.ts @@ -8,6 +8,7 @@ import {getDefaultForType} from 'app/common/gristTypes'; import {arrayRemove, arraySplice} from 'app/common/gutil'; import {SchemaTypes} from "app/common/schema"; import {UIRowId} from 'app/common/UIRowId'; +import isEqual = require('lodash/isEqual'); import fromPairs = require('lodash/fromPairs'); export interface ColTypeMap { [colId: string]: string; } @@ -330,7 +331,7 @@ export class TableData extends ActionDispatcher implements SkippableRows { return 0; } return this._rowIdCol.find((id, i) => - props.every((p) => (p.col.values[i] === p.value)) + props.every((p) => isEqual(p.col.values[i], p.value)) ) || 0; } @@ -471,7 +472,7 @@ export class TableData extends ActionDispatcher implements SkippableRows { const props = Object.keys(properties).map(p => ({col: this._columns.get(p)!, value: properties[p]})); this._rowIdCol.forEach((id, i) => { // Collect the indices of the matching rows. - if (props.every((p) => (p.col.values[i] === p.value))) { + if (props.every((p) => isEqual(p.col.values[i], p.value))) { rowIndices.push(i); } }); diff --git a/app/common/ValueParser.ts b/app/common/ValueParser.ts index fd9eb8ed..e4169167 100644 --- a/app/common/ValueParser.ts +++ b/app/common/ValueParser.ts @@ -6,12 +6,13 @@ import {safeJsonParse} from 'app/common/gutil'; import {getCurrency, NumberFormatOptions} from 'app/common/NumberFormat'; import NumberParse from 'app/common/NumberParse'; import {parseDateStrict, parseDateTime} from 'app/common/parseDate'; +import {TableData} from 'app/common/TableData'; import {DateFormatOptions, DateTimeFormatOptions, formatDecoded, FormatOptions} from 'app/common/ValueFormatter'; import flatMap = require('lodash/flatMap'); export class ValueParser { - constructor(public type: string, public widgetOpts: object, public docSettings: DocumentSettings) { + constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) { } public cleanParse(value: string): any { @@ -95,19 +96,132 @@ class ChoiceListParser extends ValueParser { } } -const parsers: { [type: string]: typeof ValueParser } = { +/** + * This is different from other widget options which are simple JSON + * stored on the field. These have to be specially derived + * for referencing columns. See ViewFieldRec.valueParser for an example. + */ +interface ReferenceParsingOptions { + visibleColId: string; + visibleColType: string; + visibleColWidgetOpts: FormatOptions; + + // If this is provided and loaded, the ValueParser will look up values directly. + // Otherwise an encoded lookup will be produced for the data engine to handle. + tableData?: TableData; +} + +export class ReferenceParser extends ValueParser { + public widgetOpts: ReferenceParsingOptions; + + protected _visibleColId = this.widgetOpts.visibleColId; + protected _tableData = this.widgetOpts.tableData; + protected _visibleColParser = createParser( + this.widgetOpts.visibleColType, + this.widgetOpts.visibleColWidgetOpts, + this.docSettings, + ); + + public parse(raw: string): any { + let value = this._visibleColParser(raw); + if (!value || !raw) { + return 0; // default value for a reference column + } + + if (this._visibleColId === 'id') { + const n = Number(value); + if (Number.isInteger(n)) { + value = n; + // Don't return yet because we need to check that this row ID exists + } else { + return raw; + } + } + + if (!this._tableData?.isLoaded) { + const options: { column: string, raw?: string } = {column: this._visibleColId}; + if (value !== raw) { + options.raw = raw; + } + return ['l', value, options]; + } + + return this._tableData.findMatchingRowId({[this._visibleColId]: value}) || raw; + } +} + +export class ReferenceListParser extends ReferenceParser { + public parse(raw: string): any { + let values: any[] | null; + try { + values = JSON.parse(raw); + } catch { + values = null; + } + if (!Array.isArray(values)) { + // csvDecodeRow should never raise an exception + values = csvDecodeRow(raw); + } + values = values.map(v => typeof v === "string" ? this._visibleColParser(v) : v); + + if (!values.length || !raw) { + return null; // null is the default value for a reference list column + } + + if (this._visibleColId === 'id') { + const numbers = values.map(Number); + if (numbers.every(Number.isInteger)) { + values = numbers; + // Don't return yet because we need to check that these row IDs exist + } else { + return raw; + } + } + + if (!this._tableData?.isLoaded) { + const options: { column: string, raw?: string } = {column: this._visibleColId}; + if (!(values.length === 1 && values[0] === raw)) { + options.raw = raw; + } + return ['l', values, options]; + } + + const rowIds: number[] = []; + for (const value of values) { + const rowId = this._tableData.findMatchingRowId({[this._visibleColId]: value}); + if (rowId) { + rowIds.push(rowId); + } else { + // There's no matching value in the visible column, i.e. this is not a valid reference. + // We need to return a string which will become AltText. + return raw; + } + } + return ['L', ...rowIds]; + } +} + +export const valueParserClasses: { [type: string]: typeof ValueParser } = { Numeric: NumericParser, Int: NumericParser, Date: DateParser, DateTime: DateTimeParser, ChoiceList: ChoiceListParser, + Ref: ReferenceParser, + RefList: ReferenceListParser, }; +/** + * Returns a function which can parse strings into values appropriate for + * a specific widget field or table column. + * widgetOpts is usually the field/column's widgetOptions JSON + * but referencing columns need more than that, see ReferenceParsingOptions above. + */ export function createParser( type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings ): (value: string) => any { - const cls = parsers[gristTypes.extractTypeFromColType(type)]; + const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)]; if (cls) { const parser = new cls(type, widgetOpts, docSettings); return parser.cleanParse.bind(parser);