(core) Move most of the reference parsing code into common so that the server can use it

Summary: Refactoring in preparation for parsing strings from the API. The plan is that the API code will only need to do a server-side version of the code in ViewFieldRec.valueParser (minus ReferenceUtils) which is quite minimal.

Test Plan: Nothing extra here, I don't think it's needed. This stuff will get tested more in a future diff which changes the API.

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D3164

app/client/lib/ReferenceUtils.ts

@@ -1,10 +1,9 @@
-import { DocData } from 'app/client/models/DocData';
+import {DocData} from 'app/client/models/DocData';
 import {ColumnRec} from 'app/client/models/entities/ColumnRec';
 import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec';
-import {SearchFunc, TableData} from 'app/client/models/TableData';
+import {TableData} from 'app/client/models/TableData';
 import {getReferencedTableId, isRefListType} from 'app/common/gristTypes';
 import {BaseFormatter} from 'app/common/ValueFormatter';
-import isEqual = require('lodash/isEqual');
 
 /**
  * Utilities for common operations involving Ref[List] fields.
@@ -40,80 +39,6 @@ export class ReferenceUtils {
     this.isRefList = isRefListType(colType);
   }
 
-  public parseReference(
-    raw: string, value: unknown
-  ): number | string | ['l', unknown, {raw?: string, column: string}] {
-    if (!value || !raw) {
-      return 0;  // default value for a reference column
-    }
-
-    if (this.visibleColId === 'id') {
-      const n = Number(value);
-      if (Number.isInteger(n)) {
-        value = n;
-      } else {
-        return raw;
-      }
-    }
-
-    if (!this.tableData.isLoaded) {
-      const options: {column: string, raw?: string} = {column: this.visibleColId};
-      if (value !== raw) {
-        options.raw = raw;
-      }
-      return ['l', value, options];
-    }
-
-    const searchFunc: SearchFunc = (v: any) => isEqual(v, value);
-    const matches = this.tableData.columnSearch(this.visibleColId, searchFunc, 1);
-    if (matches.length > 0) {
-      return matches[0];
-    } else {
-      // There's no matching value in the visible column, i.e. this is not a valid reference.
-      // We need to return a string which will become AltText.
-      return raw;
-    }
-  }
-
-  public parseReferenceList(
-    raw: string, values: unknown[]
-  ): ['L', ...number[]] | null | string | ['l', unknown[], {raw?: string, column: string}] {
-    if (!values.length || !raw) {
-      return null;  // default value for a reference list column
-    }
-
-    if (this.visibleColId === 'id') {
-      const numbers = values.map(Number);
-      if (numbers.every(Number.isInteger)) {
-        values = numbers;
-      } else {
-        return raw;
-      }
-    }
-
-    if (!this.tableData.isLoaded) {
-      const options: {column: string, raw?: string} = {column: this.visibleColId};
-      if (!(values.length === 1 && values[0] === raw)) {
-        options.raw = raw;
-      }
-      return ['l', values, options];
-    }
-
-    const rowIds: number[] = [];
-    for (const value of values) {
-      const searchFunc: SearchFunc = (v: any) => isEqual(v, value);
-      const matches = this.tableData.columnSearch(this.visibleColId, searchFunc, 1);
-      if (matches.length > 0) {
-        rowIds.push(matches[0]);
-      } else {
-        // There's no matching value in the visible column, i.e. this is not a valid reference.
-        // We need to return a string which will become AltText.
-        return raw;
-      }
-    }
-    return ['L', ...rowIds];
-  }
-
   public idToText(value: unknown) {
     if (typeof value === 'number') {
       return this.formatter.formatAny(this.tableData.getValue(value, this.visibleColId));

app/client/models/TableData.ts

@@ -10,8 +10,6 @@ import { countIf } from 'app/common/gutil';
 import { TableData as BaseTableData, ColTypeMap } from 'app/common/TableData';
 import { Emitter } from 'grainjs';
 
-export type SearchFunc = (value: any) => boolean;
-
 /**
  * TableData class to maintain a single table's data.
  */
@@ -60,33 +58,6 @@ export class TableData extends BaseTableData {
     this.dataLoadedEmitter.emit(rowIds, []);
   }
 
-  /**
-   * Given a colId and a search function, returns a list of matching row IDs, optionally limiting their number.
-   * @param {String} colId: identifies the column to search.
-   * @param {Function} searchFunc: A function which, given a column value, returns whether to include it.
-   * @param [Number] optMaxResults: if given, limit the number of returned results to this.
-   * @returns Array[Number] array of row IDs.
-   */
-  public columnSearch(colId: string, searchFunc: SearchFunc, optMaxResults?: number) {
-    const maxResults = optMaxResults || Number.POSITIVE_INFINITY;
-
-    const rowIds = this.getRowIds();
-    const valColumn = this.getColValues(colId);
-    const ret = [];
-    if (!valColumn) {
-      // tslint:disable-next-line:no-console
-      console.warn(`TableData.columnSearch called on invalid column ${this.tableId}.${colId}`);
-    } else {
-      for (let i = 0; i < rowIds.length && ret.length < maxResults; i++) {
-        const value = valColumn[i];
-        if (value && searchFunc(value)) {
-          ret.push(rowIds[i]);
-        }
-      }
-    }
-    return ret;
-  }
-
   /**
    * Counts and returns the number of error values in the given column. The count is cached to
    * keep it faster for large tables, and the cache is cleared as needed on changes to the table.

app/client/models/entities/ViewFieldRec.ts

@@ -1,10 +1,8 @@
-import {ReferenceUtils} from 'app/client/lib/ReferenceUtils';
 import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel';
 import * as modelUtil from 'app/client/models/modelUtil';
 import * as UserType from 'app/client/widgets/UserType';
-import {csvDecodeRow} from 'app/common/csvFormat';
 import {DocumentSettings} from 'app/common/DocumentSettings';
-import {isFullReferencingType} from 'app/common/gristTypes';
+import {getReferencedTableId, isFullReferencingType} from 'app/common/gristTypes';
 import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter';
 import {createParser} from 'app/common/ValueParser';
 import * as ko from 'knockout';
@@ -180,31 +178,15 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void
     const docSettings = this.documentSettings();
     const type = this.column().type();
 
-    if (!isFullReferencingType(type)) {
+    const widgetOpts = this.widgetOptionsJson();
-      return createParser(type, this.widgetOptionsJson(), docSettings);
+    if (isFullReferencingType(type)) {
-    } else {
       const vcol = this.visibleColModel();
-      const vcolParser = createParser(vcol.type(), vcol.widgetOptionsJson(), docSettings);
+      widgetOpts.visibleColId = vcol.colId() || 'id';
-      const refUtils = new ReferenceUtils(this, docModel.docData);  // uses several more observables immediately
+      widgetOpts.visibleColType = vcol.type();
-      if (!refUtils.isRefList) {
+      widgetOpts.visibleColWidgetOpts = vcol.widgetOptionsJson();
-        return (s: string) => refUtils.parseReference(s, vcolParser(s));
+      widgetOpts.tableData = docModel.docData.getTable(getReferencedTableId(type)!);
-      } else {
-        return (s: string) => {
-          let values: any[] | null;
-          try {
-            values = JSON.parse(s);
-          } catch {
-            values = null;
-          }
-          if (!Array.isArray(values)) {
-            // csvDecodeRow should never raise an exception
-            values = csvDecodeRow(s);
-          }
-          values = values.map(v => typeof v === "string" ? vcolParser(v) : v);
-          return refUtils.parseReferenceList(s, values);
-        };
-      }
     }
+    return createParser(type, widgetOpts, docSettings);
   });
 
   // The widgetOptions to read and write: either the column's or the field's own.

app/common/TableData.ts

@@ -8,6 +8,7 @@ import {getDefaultForType} from 'app/common/gristTypes';
 import {arrayRemove, arraySplice} from 'app/common/gutil';
 import {SchemaTypes} from "app/common/schema";
 import {UIRowId} from 'app/common/UIRowId';
+import isEqual = require('lodash/isEqual');
 import fromPairs = require('lodash/fromPairs');
 
 export interface ColTypeMap { [colId: string]: string; }
@@ -330,7 +331,7 @@ export class TableData extends ActionDispatcher implements SkippableRows {
       return 0;
     }
     return this._rowIdCol.find((id, i) =>
-      props.every((p) => (p.col.values[i] === p.value))
+      props.every((p) => isEqual(p.col.values[i], p.value))
     ) || 0;
   }
 
@@ -471,7 +472,7 @@ export class TableData extends ActionDispatcher implements SkippableRows {
     const props = Object.keys(properties).map(p => ({col: this._columns.get(p)!, value: properties[p]}));
     this._rowIdCol.forEach((id, i) => {
       // Collect the indices of the matching rows.
-      if (props.every((p) => (p.col.values[i] === p.value))) {
+      if (props.every((p) => isEqual(p.col.values[i], p.value))) {
         rowIndices.push(i);
       }
     });

app/common/ValueParser.ts

@@ -6,12 +6,13 @@ import {safeJsonParse} from 'app/common/gutil';
 import {getCurrency, NumberFormatOptions} from 'app/common/NumberFormat';
 import NumberParse from 'app/common/NumberParse';
 import {parseDateStrict, parseDateTime} from 'app/common/parseDate';
+import {TableData} from 'app/common/TableData';
 import {DateFormatOptions, DateTimeFormatOptions, formatDecoded, FormatOptions} from 'app/common/ValueFormatter';
 import flatMap = require('lodash/flatMap');
 
 
 export class ValueParser {
-  constructor(public type: string, public widgetOpts: object, public docSettings: DocumentSettings) {
+  constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) {
   }
 
   public cleanParse(value: string): any {
@@ -95,19 +96,132 @@ class ChoiceListParser extends ValueParser {
   }
 }
 
-const parsers: { [type: string]: typeof ValueParser } = {
+/**
+ * This is different from other widget options which are simple JSON
+ * stored on the field. These have to be specially derived
+ * for referencing columns. See ViewFieldRec.valueParser for an example.
+ */
+interface ReferenceParsingOptions {
+  visibleColId: string;
+  visibleColType: string;
+  visibleColWidgetOpts: FormatOptions;
+
+  // If this is provided and loaded, the ValueParser will look up values directly.
+  // Otherwise an encoded lookup will be produced for the data engine to handle.
+  tableData?: TableData;
+}
+
+export class ReferenceParser extends ValueParser {
+  public widgetOpts: ReferenceParsingOptions;
+
+  protected _visibleColId = this.widgetOpts.visibleColId;
+  protected _tableData = this.widgetOpts.tableData;
+  protected _visibleColParser = createParser(
+    this.widgetOpts.visibleColType,
+    this.widgetOpts.visibleColWidgetOpts,
+    this.docSettings,
+  );
+
+  public parse(raw: string): any {
+    let value = this._visibleColParser(raw);
+    if (!value || !raw) {
+      return 0;  // default value for a reference column
+    }
+
+    if (this._visibleColId === 'id') {
+      const n = Number(value);
+      if (Number.isInteger(n)) {
+        value = n;
+        // Don't return yet because we need to check that this row ID exists
+      } else {
+        return raw;
+      }
+    }
+
+    if (!this._tableData?.isLoaded) {
+      const options: { column: string, raw?: string } = {column: this._visibleColId};
+      if (value !== raw) {
+        options.raw = raw;
+      }
+      return ['l', value, options];
+    }
+
+    return this._tableData.findMatchingRowId({[this._visibleColId]: value}) || raw;
+  }
+}
+
+export class ReferenceListParser extends ReferenceParser {
+  public parse(raw: string): any {
+    let values: any[] | null;
+    try {
+      values = JSON.parse(raw);
+    } catch {
+      values = null;
+    }
+    if (!Array.isArray(values)) {
+      // csvDecodeRow should never raise an exception
+      values = csvDecodeRow(raw);
+    }
+    values = values.map(v => typeof v === "string" ? this._visibleColParser(v) : v);
+
+    if (!values.length || !raw) {
+      return null;  // null is the default value for a reference list column
+    }
+
+    if (this._visibleColId === 'id') {
+      const numbers = values.map(Number);
+      if (numbers.every(Number.isInteger)) {
+        values = numbers;
+        // Don't return yet because we need to check that these row IDs exist
+      } else {
+        return raw;
+      }
+    }
+
+    if (!this._tableData?.isLoaded) {
+      const options: { column: string, raw?: string } = {column: this._visibleColId};
+      if (!(values.length === 1 && values[0] === raw)) {
+        options.raw = raw;
+      }
+      return ['l', values, options];
+    }
+
+    const rowIds: number[] = [];
+    for (const value of values) {
+      const rowId = this._tableData.findMatchingRowId({[this._visibleColId]: value});
+      if (rowId) {
+        rowIds.push(rowId);
+      } else {
+        // There's no matching value in the visible column, i.e. this is not a valid reference.
+        // We need to return a string which will become AltText.
+        return raw;
+      }
+    }
+    return ['L', ...rowIds];
+  }
+}
+
+export const valueParserClasses: { [type: string]: typeof ValueParser } = {
   Numeric: NumericParser,
   Int: NumericParser,
   Date: DateParser,
   DateTime: DateTimeParser,
   ChoiceList: ChoiceListParser,
+  Ref: ReferenceParser,
+  RefList: ReferenceListParser,
 };
 
 
+/**
+ * Returns a function which can parse strings into values appropriate for
+ * a specific widget field or table column.
+ * widgetOpts is usually the field/column's widgetOptions JSON
+ * but referencing columns need more than that, see ReferenceParsingOptions above.
+ */
 export function createParser(
   type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings
 ): (value: string) => any {
-  const cls = parsers[gristTypes.extractTypeFromColType(type)];
+  const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)];
   if (cls) {
     const parser = new cls(type, widgetOpts, docSettings);
     return parser.cleanParse.bind(parser);