(core) New type conversion in the backend

Summary: This is https://phab.getgrist.com/D3205 plus some changes (https://github.com/dsagal/grist/compare/type-convert...type-convert-server?expand=1) that move the conversion process to the backend. A new user action ConvertFromColumn uses `call_external` so that the data engine can delegate back to ActiveDoc. Code for creating formatters and parsers is significantly refactored so that most of the logic is in `common` and can be used in different ways.

Test Plan: The original diff adds plenty of tests.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Subscribers: dsagal

Differential Revision: https://phab.getgrist.com/D3240

app/client/models/DocModel.ts

@@ -22,7 +22,7 @@ import {urlState} from 'app/client/models/gristUrlState';
 import * as MetaRowModel from 'app/client/models/MetaRowModel';
 import * as MetaTableModel from 'app/client/models/MetaTableModel';
 import * as rowset from 'app/client/models/rowset';
-import {isHiddenTable} from 'app/client/models/isHiddenTable';
+import {isHiddenTable} from 'app/common/isHiddenTable';
 import {schema, SchemaTypes} from 'app/common/schema';
 
 import {ACLRuleRec, createACLRuleRec} from 'app/client/models/entities/ACLRuleRec';

app/client/models/entities/ColumnRec.ts

@@ -2,8 +2,13 @@ import {KoArray} from 'app/client/lib/koArray';
 import {DocModel, IRowModel, recordSet, refRecord, TableRec, ViewFieldRec} from 'app/client/models/DocModel';
 import {jsonObservable, ObjObservable} from 'app/client/models/modelUtil';
 import * as gristTypes from 'app/common/gristTypes';
-import {getReferencedTableId, isFullReferencingType} from 'app/common/gristTypes';
-import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter';
+import {getReferencedTableId} from 'app/common/gristTypes';
+import {
+  BaseFormatter,
+  createFullFormatterRaw,
+  createVisibleColFormatterRaw,
+  FullFormatterArgs
+} from 'app/common/ValueFormatter';
 import * as ko from 'knockout';
 
 // Represents a column in a user-defined table.
@@ -124,38 +129,23 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void {
 
   // Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol
   // associated with this column. If no visible column available, return formatting for the column itself.
-  this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this, docModel));
+  this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'vcol'));
 
-  this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, this.visibleColFormatter()));
-}
-
-export function visibleColFormatterForRec(
-  rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel
-): BaseFormatter {
-  const vcol = rec.visibleColModel();
-  const documentSettings = docModel.docInfoRow.documentSettingsJson();
-  const type = colRec.type();
-  if (isFullReferencingType(type)) {
-    if (vcol.getRowId() === 0) {
-      // This column displays the Row ID, e.g. Table1[2]
-      // referencedTableId may actually be empty if the table is hidden
-      const referencedTableId: string = colRec.refTable()?.tableId() || "";
-      return createFormatter('Id', {tableId: referencedTableId}, documentSettings);
-    } else {
-      return createFormatter(vcol.type(), vcol.widgetOptionsJson(), documentSettings);
-    }
-  } else {
-    // For non-reference columns, there's no 'visible column' and we just return a regular formatter
-    return createFormatter(type, rec.widgetOptionsJson(), documentSettings);
-  }
+  this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'full'));
 }
 
 export function formatterForRec(
-  rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, visibleColFormatter: BaseFormatter
+  rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, kind: 'full' | 'vcol'
 ): BaseFormatter {
-  const type = colRec.type();
-  // Ref/RefList columns delegate most formatting to the visibleColFormatter
-  const widgetOpts = {...rec.widgetOptionsJson(), visibleColFormatter};
-  const documentSettings = docModel.docInfoRow.documentSettingsJson();
-  return createFormatter(type, widgetOpts, documentSettings);
+  const vcol = rec.visibleColModel();
+  const func = kind === 'full' ? createFullFormatterRaw : createVisibleColFormatterRaw;
+  const args: FullFormatterArgs = {
+    docData: docModel.docData,
+    type: colRec.type(),
+    widgetOpts: rec.widgetOptionsJson(),
+    visibleColType: vcol?.type(),
+    visibleColWidgetOpts: vcol?.widgetOptionsJson(),
+    docSettings: docModel.docInfoRow.documentSettingsJson(),
+  };
+  return func(args);
 }

app/client/models/entities/ViewFieldRec.ts

@@ -1,5 +1,5 @@
 import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel';
-import {formatterForRec, visibleColFormatterForRec} from 'app/client/models/entities/ColumnRec';
+import {formatterForRec} from 'app/client/models/entities/ColumnRec';
 import * as modelUtil from 'app/client/models/modelUtil';
 import * as UserType from 'app/client/widgets/UserType';
 import {DocumentSettings} from 'app/common/DocumentSettings';
@@ -172,13 +172,14 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void
 
   // Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol
   // associated with this field. If no visible column available, return formatting for the field itself.
-  this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this.column(), docModel));
+  this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'vcol'));
 
-  this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, this.visibleColFormatter()));
+  this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'full'));
 
   this.createValueParser = function() {
     const fieldRef = this.useColOptions.peek() ? undefined : this.id.peek();
-    return createParser(docModel.docData, this.colRef.peek(), fieldRef);
+    const parser = createParser(docModel.docData, this.colRef.peek(), fieldRef);
+    return parser.cleanParse.bind(parser);
   };
 
   // The widgetOptions to read and write: either the column's or the field's own.

app/client/models/isHiddenTable.ts

@@ -1,12 +0,0 @@
-import {RowId} from 'app/client/models/rowset';
-import {TableData} from 'app/client/models/TableData';
-
-/**
- * Return whether a table identified by the rowId of its metadata record, should normally be
- * hidden from the user (e.g. as an option in the page-widget picker).
- */
-export function isHiddenTable(tablesData: TableData, tableRef: RowId): boolean {
-  const tableId = tablesData.getValue(tableRef, 'tableId') as string|undefined;
-  return tablesData.getValue(tableRef, 'summarySourceTable') !== 0 ||
-    Boolean(tableId?.startsWith('GristHidden'));
-}