(core) New type conversion in the backend

Summary: This is https://phab.getgrist.com/D3205 plus some changes (https://github.com/dsagal/grist/compare/type-convert...type-convert-server?expand=1) that move the conversion process to the backend. A new user action ConvertFromColumn uses `call_external` so that the data engine can delegate back to ActiveDoc. Code for creating formatters and parsers is significantly refactored so that most of the logic is in `common` and can be used in different ways.

Test Plan: The original diff adds plenty of tests.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Subscribers: dsagal

Differential Revision: https://phab.getgrist.com/D3240

app/common/ValueConverter.ts

@@ -0,0 +1,255 @@
+import {DocData} from 'app/common/DocData';
+import * as gristTypes from 'app/common/gristTypes';
+import {isList} from 'app/common/gristTypes';
+import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFormatter';
+import {
+  createParserOrFormatterArgumentsRaw,
+  createParserRaw,
+  ReferenceListParser,
+  ReferenceParser,
+  ValueParser
+} from 'app/common/ValueParser';
+import {CellValue, GristObjCode} from 'app/plugin/GristData';
+
+
+/**
+ * Base class for converting values from one type to another with the convert() method.
+ * Has a formatter for the source column
+ * and a parser for the destination column.
+ *
+ * The default convert() is for non-list destination types, so if the source value
+ * is a list it only converts nicely if the list contains exactly one element.
+ */
+export class ValueConverter {
+  constructor(public formatter: BaseFormatter, public parser: ValueParser) {
+  }
+
+  public convert(value: any): any {
+    if (isList(value)) {
+      if (value.length === 1) {
+        // Empty list: ['L']
+        return null;
+      } else if (value.length === 2) {
+        // Singleton list: ['L', value]
+        // Convert just that one value.
+        value = value[1];
+      } else {
+        // List with multiple values. Since we're converting to just one value,
+        // format the whole thing as text, which is an error for most types.
+        return this.formatter.formatAny(value);
+      }
+    }
+    return this.convertInner(value);
+  }
+
+  protected convertInner(value: any): any {
+    const formatted = this.formatter.formatAny(value);
+    return this.parser.cleanParse(formatted);
+  }
+}
+
+/**
+ * Base class for converting to a list type (Reference List or Choice List).
+ *
+ * Wraps single values in a list, and converts lists elementwise.
+ */
+class ListConverter extends ValueConverter {
+  // Don't parse strings like "Smith, John" which may look like lists but represent a single choice.
+  // TODO this works when the source is a Choice column, but not when it's a Reference to a Choice column.
+  //   But the guessed choices are also broken in that case.
+  private _choices: Set<string> = new Set((this.formatter.widgetOpts as any).choices || []);
+
+  public convert(value: any): any {
+    if (typeof value === "string" && !this._choices.has(value)) {
+      // Parse CSV/JSON
+      return this.parser.cleanParse(value);
+    }
+    const values = isList(value) ? value.slice(1) : [value];
+    if (!values.length || value == null) {
+      return null;
+    }
+    return this.handleValues(value, values.map(v => this.convertInner(v)));
+  }
+
+  protected handleValues(originalValue: any, values: any[]) {
+    return ['L', ...values];
+  }
+}
+
+class ChoiceListConverter extends ListConverter {
+  /**
+   * Convert each source value to a 'Choice'
+   */
+  protected convertInner(value: any): any {
+    return this.formatter.formatAny(value);
+  }
+}
+
+class ReferenceListConverter extends ListConverter {
+  private _innerConverter = new ReferenceConverter(
+    this.formatter,
+    new ReferenceParser("Ref", this.parser.widgetOpts, this.parser.docSettings),
+  );
+
+  constructor(public formatter: BaseFormatter, public parser: ReferenceListParser) {
+    super(formatter, parser);
+    // Prevent the parser from looking up reference values in the frontend.
+    // Leave it to the data engine which has a much more efficient algorithm for long lists of values.
+    delete parser.tableData;
+  }
+
+  public handleValues(originalValue: any, values: any[]): any {
+    const result = [];
+    let lookupColumn: string = "";
+    const raw = this.formatter.formatAny(originalValue);  // AltText if the reference lookup fails
+    for (const value of values) {
+      if (typeof value === "string") {
+        // Failed to parse one of the references, so return a raw string for the whole thing
+        return raw;
+      } else {
+        // value is a lookup tuple: ['l', value, options]
+        result.push(value[1]);
+        lookupColumn = value[2].column;
+      }
+    }
+    return ['l', result, {column: lookupColumn, raw}];
+  }
+
+  /**
+   * Convert each source value to a 'Reference'
+   */
+  protected convertInner(value: any): any {
+    return this._innerConverter.convert(value);
+  }
+}
+
+class ReferenceConverter extends ValueConverter {
+  private _innerConverter: ValueConverter = createConverter(this.formatter, this.parser.visibleColParser);
+
+  constructor(public formatter: BaseFormatter, public parser: ReferenceParser) {
+    super(formatter, parser);
+    // Prevent the parser from looking up reference values in the frontend.
+    // Leave it to the data engine which has a much more efficient algorithm for long lists of values.
+    delete parser.tableData;
+  }
+
+  protected convertInner(value: any): any {
+    // Convert to the type of the visible column.
+    const converted = this._innerConverter.convert(value);
+    return this.parser.lookup(converted, this.formatter.formatAny(value));
+  }
+}
+
+class NumericConverter extends ValueConverter {
+  protected convertInner(value: any): any {
+    if (typeof value === "boolean") {
+      return value ? 1 : 0;
+    }
+    return super.convertInner(value);
+  }
+}
+
+class DateConverter extends ValueConverter {
+  private _sourceType = gristTypes.extractInfoFromColType(this.formatter.type);
+
+  protected convertInner(value: any): any {
+    // When converting Date->DateTime, DateTime->Date, or between DateTime timezones,
+    // it's important to send an encoded Date/DateTime object rather than just a timestamp number
+    // so that the data engine knows what to do in do_convert, especially regarding timezones.
+    // If the source column is a Reference to a Date/DateTime then `value` is already
+    // an encoded object from the display column which has type Any.
+    value = gristTypes.reencodeAsAny(value, this._sourceType);
+    if (Array.isArray(value) && (
+      value[0] === GristObjCode.Date ||
+      value[0] === GristObjCode.DateTime
+    )) {
+      return value;
+    }
+    return super.convertInner(value);
+  }
+}
+
+export const valueConverterClasses: { [type: string]: typeof ValueConverter } = {
+  Date: DateConverter,
+  DateTime: DateConverter,
+  ChoiceList: ChoiceListConverter,
+  Ref: ReferenceConverter,
+  RefList: ReferenceListConverter,
+  Numeric: NumericConverter,
+  Int: NumericConverter,
+};
+
+export function createConverter(formatter: BaseFormatter, parser: ValueParser) {
+  const cls = valueConverterClasses[gristTypes.extractTypeFromColType(parser.type)] || ValueConverter;
+  return new cls(formatter, parser);
+}
+
+/**
+ * Used by the ConvertFromColumn user action in the data engine.
+ * The higher order function separates docData (passed by ActiveDoc)
+ * from the arguments passed to call_external in Python.
+ */
+export function convertFromColumn(docData: DocData) {
+  return function(
+    sourceColRef: number,
+    type: string,
+    widgetOpts: string,
+    visibleColRef: number,
+    values: ReadonlyArray<CellValue>,
+    displayColValues?: ReadonlyArray<CellValue>,
+  ): CellValue[] {
+    const formatter = createFullFormatterFromDocData(docData, sourceColRef);
+    const parser = createParserRaw(
+      ...createParserOrFormatterArgumentsRaw(docData, type, widgetOpts, visibleColRef)
+    );
+    const converter = createConverter(formatter, parser);
+    return convertValues(converter, values, displayColValues || values);
+  };
+}
+
+export function convertValues(
+  converter: ValueConverter,
+  // Raw values from the actual column, e.g. row IDs for reference columns
+  values: ReadonlyArray<CellValue>,
+  // Values from the display column, which is the same as the raw values for non-referencing columns.
+  // In almost all cases these are the values that actually matter and get converted.
+  displayColValues: ReadonlyArray<CellValue>,
+): CellValue[] {
+  // Converting Ref <-> RefList without changing the target table is a special case - see prepTransformColInfo.
+  // In this case we deal with the actual row IDs stored in the real column,
+  // whereas in all other cases we use display column values.
+  const sourceType = gristTypes.extractInfoFromColType(converter.formatter.type);
+  const targetType = gristTypes.extractInfoFromColType(converter.parser.type);
+  const refToRefList = (
+    sourceType.type === "Ref" &&
+    targetType.type === "RefList" &&
+    sourceType.tableId === targetType.tableId
+  );
+  const refListToRef = (
+    sourceType.type === "RefList" &&
+    targetType.type === "Ref" &&
+    sourceType.tableId === targetType.tableId
+  );
+
+  return displayColValues.map((displayVal, i) => {
+    const actualValue = values[i];
+
+    if (refToRefList && typeof actualValue === "number") {
+      if (actualValue === 0) {
+        return null;
+      } else {
+        return ["L", actualValue];
+      }
+    } else if (refListToRef && isList(actualValue)) {
+      if (actualValue.length === 1) {
+        // Empty list: ['L']
+        return 0;
+      } else if (actualValue.length === 2) {
+        // Singleton list: ['L', rowId]
+        return actualValue[1];
+      }
+    }
+
+    return converter.convert(displayVal);
+  });
+}

app/common/ValueFormatter.ts

@@ -2,11 +2,14 @@
 
 import {csvEncodeRow} from 'app/common/csvFormat';
 import {CellValue} from 'app/common/DocActions';
+import {DocData} from 'app/common/DocData';
 import {DocumentSettings} from 'app/common/DocumentSettings';
-import {getReferencedTableId, isList} from 'app/common/gristTypes';
 import * as gristTypes from 'app/common/gristTypes';
+import {getReferencedTableId, isList} from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
+import {isHiddenTable} from 'app/common/isHiddenTable';
 import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
+import {createParserOrFormatterArguments, ReferenceParsingOptions} from 'app/common/ValueParser';
 import {GristObjCode} from 'app/plugin/GristData';
 import {decodeObject, GristDateTime} from 'app/plugin/objtypes';
 import * as moment from 'moment-timezone';
@@ -280,3 +283,70 @@ export function createFormatter(type: string, widgetOpts: FormatOptions, docSett
   const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter;
   return new ctor(type, widgetOpts, docSettings);
 }
+
+export interface FullFormatterArgs {
+  docData: DocData;
+  type: string;
+  widgetOpts: FormatOptions;
+  visibleColType: string;
+  visibleColWidgetOpts: FormatOptions;
+  docSettings: DocumentSettings;
+}
+
+/**
+ * Returns a constructor
+ * with a format function that can properly convert a value passed to it into the
+ * right format for that column.
+ *
+ * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
+ * instead of the table column.
+ */
+export function createFullFormatterFromDocData(
+  docData: DocData,
+  colRef: number,
+  fieldRef?: number,
+): BaseFormatter {
+  const [type, widgetOpts, docSettings] = createParserOrFormatterArguments(docData, colRef, fieldRef);
+  const {visibleColType, visibleColWidgetOpts} = widgetOpts as ReferenceParsingOptions;
+  return createFullFormatterRaw({
+    docData,
+    type,
+    widgetOpts,
+    visibleColType,
+    visibleColWidgetOpts,
+    docSettings,
+  });
+}
+
+export function createFullFormatterRaw(args: FullFormatterArgs) {
+  const {type, widgetOpts, docSettings} = args;
+  const visibleColFormatter = createVisibleColFormatterRaw(args);
+  return createFormatter(type, {...widgetOpts, visibleColFormatter}, docSettings);
+}
+
+export function createVisibleColFormatterRaw(
+  {
+    docData,
+    docSettings,
+    type,
+    visibleColType,
+    visibleColWidgetOpts,
+    widgetOpts
+  }: FullFormatterArgs
+): BaseFormatter {
+  let referencedTableId = gristTypes.getReferencedTableId(type);
+  if (!referencedTableId) {
+    return createFormatter(type, widgetOpts, docSettings);
+  } else if (visibleColType) {
+    return createFormatter(visibleColType, visibleColWidgetOpts, docSettings);
+  } else {
+    // This column displays the Row ID, e.g. Table1[2]
+    // Make referencedTableId empty if the table is hidden
+    const tablesData = docData.getMetaTable("_grist_Tables");
+    const tableRef = tablesData.findRow("tableId", referencedTableId);
+    if (isHiddenTable(tablesData, tableRef)) {
+      referencedTableId = "";
+    }
+    return createFormatter('Id', {tableId: referencedTableId}, docSettings);
+  }
+}

app/common/ValueParser.ts

@@ -33,6 +33,9 @@ export class ValueParser {
 
 }
 
+class IdentityParser extends ValueParser {
+}
+
 /**
  * Same as basic Value parser, but will return null if a value is an empty string.
  */
@@ -117,7 +120,7 @@ class ChoiceListParser extends ValueParser {
  * stored on the field. These have to be specially derived
  * for referencing columns. See createParser.
  */
-interface ReferenceParsingOptions {
+export interface ReferenceParsingOptions {
   visibleColId: string;
   visibleColType: string;
   visibleColWidgetOpts: FormatOptions;
@@ -129,18 +132,22 @@ interface ReferenceParsingOptions {
 
 export class ReferenceParser extends ValueParser {
   public widgetOpts: ReferenceParsingOptions;
-
-  protected _visibleColId = this.widgetOpts.visibleColId;
-  protected _tableData = this.widgetOpts.tableData;
-  protected _visibleColParser = createParserRaw(
+  public tableData = this.widgetOpts.tableData;
+  public visibleColParser = createParserRaw(
     this.widgetOpts.visibleColType,
     this.widgetOpts.visibleColWidgetOpts,
     this.docSettings,
   );
 
+  protected _visibleColId = this.widgetOpts.visibleColId;
+
   public parse(raw: string): any {
-    let value = this._visibleColParser(raw);
-    if (!value || !raw) {
+    const value = this.visibleColParser.cleanParse(raw);
+    return this.lookup(value, raw);
+  }
+
+  public lookup(value: any, raw: string): any {
+    if (value == null || value === "" || !raw) {
       return 0;  // default value for a reference column
     }
 
@@ -154,7 +161,7 @@ export class ReferenceParser extends ValueParser {
       }
     }
 
-    if (!this._tableData?.isLoaded) {
+    if (!this.tableData?.isLoaded) {
       const options: { column: string, raw?: string } = {column: this._visibleColId};
       if (value !== raw) {
         options.raw = raw;
@@ -162,7 +169,7 @@ export class ReferenceParser extends ValueParser {
       return ['l', value, options];
     }
 
-    return this._tableData.findMatchingRowId({[this._visibleColId]: value}) || raw;
+    return this.tableData.findMatchingRowId({[this._visibleColId]: value}) || raw;
   }
 }
 
@@ -178,7 +185,7 @@ export class ReferenceListParser extends ReferenceParser {
       // csvDecodeRow should never raise an exception
       values = csvDecodeRow(raw);
     }
-    values = values.map(v => typeof v === "string" ? this._visibleColParser(v) : encodeObject(v));
+    values = values.map(v => typeof v === "string" ? this.visibleColParser.cleanParse(v) : encodeObject(v));
 
     if (!values.length || !raw) {
       return null;  // null is the default value for a reference list column
@@ -194,7 +201,7 @@ export class ReferenceListParser extends ReferenceParser {
       }
     }
 
-    if (!this._tableData?.isLoaded) {
+    if (!this.tableData?.isLoaded) {
       const options: { column: string, raw?: string } = {column: this._visibleColId};
       if (!(values.length === 1 && values[0] === raw)) {
         options.raw = raw;
@@ -204,7 +211,7 @@ export class ReferenceListParser extends ReferenceParser {
 
     const rowIds: number[] = [];
     for (const value of values) {
-      const rowId = this._tableData.findMatchingRowId({[this._visibleColId]: value});
+      const rowId = this.tableData.findMatchingRowId({[this._visibleColId]: value});
       if (rowId) {
         rowIds.push(rowId);
       } else {
@@ -228,27 +235,21 @@ export const valueParserClasses: { [type: string]: typeof ValueParser } = {
   RefList: ReferenceListParser,
 };
 
-const identity = (value: string) => value;
-
 /**
- * Returns a function which can parse strings into values appropriate for
+ * Returns a ValueParser which can parse strings into values appropriate for
  * a specific widget field or table column.
  * widgetOpts is usually the field/column's widgetOptions JSON
  * but referencing columns need more than that, see ReferenceParsingOptions above.
  */
 export function createParserRaw(
   type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings
-): (value: string) => any {
-  const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)];
-  if (cls) {
-    const parser = new cls(type, widgetOpts, docSettings);
-    return parser.cleanParse.bind(parser);
-  }
-  return identity;
+): ValueParser {
+  const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)] || IdentityParser;
+  return new cls(type, widgetOpts, docSettings);
 }
 
 /**
- * Returns a function which can parse strings into values appropriate for
+ * Returns a ValueParser which can parse strings into values appropriate for
  * a specific widget field or table column.
  *
  * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
@@ -258,23 +259,46 @@ export function createParser(
   docData: DocData,
   colRef: number,
   fieldRef?: number,
-): (value: string) => any {
+): ValueParser {
+  return createParserRaw(...createParserOrFormatterArguments(docData, colRef, fieldRef));
+}
+
+/**
+ * Returns arguments suitable for createParserRaw or createFormatter. Only for internal use.
+ *
+ * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
+ * instead of the table column.
+ */
+export function createParserOrFormatterArguments(
+  docData: DocData,
+  colRef: number,
+  fieldRef?: number,
+): [string, object, DocumentSettings] {
   const columnsTable = docData.getMetaTable('_grist_Tables_column');
   const fieldsTable = docData.getMetaTable('_grist_Views_section_field');
-  const docInfoTable = docData.getMetaTable('_grist_DocInfo');
 
   const col = columnsTable.getRecord(colRef)!;
-
   let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col;
   if (fieldRef) {
     fieldOrCol = fieldsTable.getRecord(fieldRef) || col;
   }
 
-  const widgetOpts = safeJsonParse(fieldOrCol.widgetOptions, {});
+  return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol);
+}
+
+export function createParserOrFormatterArgumentsRaw(
+  docData: DocData,
+  type: string,
+  widgetOptions: string,
+  visibleColRef: number,
+): [string, object, DocumentSettings] {
+  const columnsTable = docData.getMetaTable('_grist_Tables_column');
+  const docInfoTable = docData.getMetaTable('_grist_DocInfo');
+
+  const widgetOpts = safeJsonParse(widgetOptions, {});
 
-  const type = col.type;
   if (isFullReferencingType(type)) {
-    const vcol = columnsTable.getRecord(fieldOrCol.visibleCol);
+    const vcol = columnsTable.getRecord(visibleColRef);
     widgetOpts.visibleColId = vcol?.colId || 'id';
     widgetOpts.visibleColType = vcol?.type;
     widgetOpts.visibleColWidgetOpts = safeJsonParse(vcol?.widgetOptions || '', {});
@@ -284,7 +308,7 @@ export function createParser(
   const docInfo = docInfoTable.getRecord(1);
   const docSettings = safeJsonParse(docInfo!.documentSettings, {}) as DocumentSettings;
 
-  return createParserRaw(type, widgetOpts, docSettings);
+  return [type, widgetOpts, docSettings];
 }
 
 /**
@@ -311,12 +335,12 @@ function parseColValues<T extends ColValues | BulkColValues>(
     const parser = createParser(docData, colRef);
 
     // Optimisation: If there's no special parser for this column type, do nothing
-    if (parser === identity) {
+    if (parser instanceof IdentityParser) {
       return values;
     }
 
     function parseIfString(val: any) {
-      return typeof val === "string" ? parser(val) : val;
+      return typeof val === "string" ? parser.cleanParse(val) : val;
     }
 
     if (bulk) {

app/common/isHiddenTable.ts

@@ -0,0 +1,12 @@
+import {UIRowId} from 'app/common/UIRowId';
+import {TableData} from "./TableData";
+
+/**
+ * Return whether a table identified by the rowId of its metadata record, should normally be
+ * hidden from the user (e.g. as an option in the page-widget picker).
+ */
+export function isHiddenTable(tablesData: TableData, tableRef: UIRowId): boolean {
+  const tableId = tablesData.getValue(tableRef, 'tableId') as string|undefined;
+  return tablesData.getValue(tableRef, 'summarySourceTable') !== 0 ||
+    Boolean(tableId?.startsWith('GristHidden'));
+}