(core) New type conversion in the backend

Summary: This is https://phab.getgrist.com/D3205 plus some changes (https://github.com/dsagal/grist/compare/type-convert...type-convert-server?expand=1) that move the conversion process to the backend. A new user action ConvertFromColumn uses `call_external` so that the data engine can delegate back to ActiveDoc. Code for creating formatters and parsers is significantly refactored so that most of the logic is in `common` and can be used in different ways. Test Plan: The original diff adds plenty of tests. Reviewers: georgegevoian Reviewed By: georgegevoian Subscribers: dsagal Differential Revision: https://phab.getgrist.com/D3240
2026-03-02 04:09:24 +00:00 · 2022-02-04 13:13:03 +02:00
parent 4890a1fe89
commit 5d671bf0b3
25 changed files with 593 additions and 492 deletions
--- a/app/client/components/ColumnTransform.ts
+++ b/app/client/components/ColumnTransform.ts
@@ -142,6 +142,10 @@ export class ColumnTransform extends Disposable {
    return actions.every(action => (
      // ['AddColumn', USER_TABLE, 'gristHelper_Transform', colInfo]
      (action[2] === 'gristHelper_Transform') ||
+      // ['AddColumn', USER_TABLE, 'gristHelper_Converted', colInfo]
+      (action[2] === 'gristHelper_Converted') ||
+      // ['ConvertFromColumn', USER_TABLE, SOURCE_COLUMN, 'gristHelper_Converted']
+      (action[3] === 'gristHelper_Converted') ||
      // ["SetDisplayFormula", USER_TABLE, ...]
      (action[0] === 'SetDisplayFormula') ||
      // ['UpdateRecord', '_grist_Table_column', transformColId, ...]
@@ -192,7 +196,6 @@ export class ColumnTransform extends Disposable {
    // Define variables used after await, since this will be disposed by then.
    const transformColId = this.transformColumn.colId();
    const field = this.field;
-    const fieldBuilder = this._fieldBuilder;
    const origRef = this.origColumn.getRowId();
    const tableData = this._tableData;
    this.isCallPending(true);
@@ -201,17 +204,36 @@ export class ColumnTransform extends Disposable {
        // TODO: Values flicker during executing since transform column remains a formula as values are copied
        // back to the original column. The CopyFromColumn useraction really ought to be "CopyAndRemove" since
        // that seems the best way to avoid calculating the formula on wrong values.
-        return await tableData.sendTableAction(['CopyFromColumn', transformColId, this.origColumn.colId(),
-          JSON.stringify(fieldBuilder.options())]);
+        await this.gristDoc.docData.sendActions(this.executeActions());
      }
    } finally {
      // Wait until the change completed to set column back, to avoid value flickering.
      field.colRef(origRef);
      void tableData.sendTableAction(['RemoveColumn', transformColId]);
+      this.cleanup();
      this.dispose();
    }
  }

+  /**
+   * The user actions to send when actually executing the transform.
+   */
+  protected executeActions(): UserAction[] {
+    return [
+      [
+        'CopyFromColumn',
+        this._tableData.tableId,
+        this.transformColumn.colId(),
+        this.origColumn.colId(),
+        JSON.stringify(this._fieldBuilder.options()),
+      ],
+    ];
+  }
+
+  protected cleanup() {
+    // For overriding
+  }
+
  protected getIdentityFormula() {
    return 'return $' + this.origColumn.colId();
  }
--- a/app/client/components/TypeConversion.ts
+++ b/app/client/components/TypeConversion.ts
@@ -6,7 +6,6 @@

 import {DocModel} from 'app/client/models/DocModel';
 import {ColumnRec} from 'app/client/models/entities/ColumnRec';
-import * as UserType from 'app/client/widgets/UserType';
 import * as gristTypes from 'app/common/gristTypes';
 import {isFullReferencingType} from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
@@ -90,7 +89,7 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
    type: addColTypeSuffix(toTypeMaybeFull, origCol, docModel),
    isFormula: true,
    visibleCol: 0,
-    formula: "",          // Will be filled in at the end.
+    formula: "CURRENT_CONVERSION(rec)",
  };

  const prevOptions = origCol.widgetOptionsJson.peek() || {};
@@ -139,23 +138,35 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
    case 'RefList':
    {
      // Set suggested destination table and visible column.
-      // Null if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen).
-      const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`)!;
+      // Undefined if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen).
+      const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`) || undefined;

-      // Finds a reference suggestion column and sets it as the current reference value.
-      const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100);
-      if (!columnData) { break; }
-      columnData.delete(gristTypes.getDefaultForType(origCol.type()));
+      let suggestedColRef: number;
+      let suggestedTableId: string;
+      const origColTypeInfo = gristTypes.extractInfoFromColType(origCol.type.peek());
+      if (!optTableId && origColTypeInfo.type === "Ref" || origColTypeInfo.type === "RefList") {
+        // When converting between Ref and Reflist, initially suggest the same table and visible column.
+        // When converting, if the table is the same, it's a special case.
+        // The visible column will not affect conversion.
+        // It will simply wrap the reference (row ID) in a list or extract the one element of a reference list.
+        suggestedColRef = origCol.visibleCol.peek();
+        suggestedTableId = origColTypeInfo.tableId;
+      } else {
+        // Finds a reference suggestion column and sets it as the current reference value.
+        const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100);
+        if (!columnData) { break; }
+        columnData.delete(gristTypes.getDefaultForType(origCol.type()));

-      // 'findColFromValues' function requires an array since it sends the values to the sandbox.
-      const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId);
-      const suggestedColRef = matches.find(match => match !== origCol.getRowId());
-      if (!suggestedColRef) { break; }
-      const suggestedCol = docModel.columns.getRowModel(suggestedColRef);
-      const suggestedTableId = suggestedCol.table().tableId();
-      if (optTableId && suggestedTableId !== optTableId) {
-        console.warn("Inappropriate column received from findColFromValues");
-        break;
+        // 'findColFromValues' function requires an array since it sends the values to the sandbox.
+        const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId);
+        suggestedColRef = matches.find(match => match !== origCol.getRowId())!;
+        if (!suggestedColRef) { break; }
+        const suggestedCol = docModel.columns.getRowModel(suggestedColRef);
+        suggestedTableId = suggestedCol.table().tableId();
+        if (optTableId && suggestedTableId !== optTableId) {
+          console.warn("Inappropriate column received from findColFromValues");
+          break;
+        }
      }
      colInfo.type = `${toType}:${suggestedTableId}`;
      colInfo.visibleCol = suggestedColRef;
@@ -163,11 +174,9 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
    }
  }

-  const newOptions = UserType.mergeOptions(widgetOptions || {}, colInfo.type);
  if (widgetOptions) {
    colInfo.widgetOptions = JSON.stringify(widgetOptions);
  }
-  colInfo.formula = getDefaultFormula(docModel, origCol, colInfo.type, colInfo.visibleCol, newOptions);
  return colInfo;
 }

@@ -184,62 +193,6 @@ export async function setDisplayFormula(
  }
 }

-// Given the original column and info about the new column, returns the formula to use for the
-// transform column to do the transformation.
-export function getDefaultFormula(
-  docModel: DocModel, origCol: ColumnRec, newType: string,
-  newVisibleCol: number, newWidgetOptions: any): string {
-
-  const colId = origCol.colId();
-  const oldVisibleColName = isReferenceCol(origCol) ?
-    getVisibleColName(docModel, origCol.visibleCol()) : undefined;
-
-  let origValFormula = oldVisibleColName ?
-    // The `str()` below converts AltText to plain text.
-    `($${colId}.${oldVisibleColName}
-    if ISREF($${colId}) or ISREFLIST($${colId})
-    else str($${colId}))`
-    : `$${colId}`;
-
-  if (origCol.type.peek() === 'ChoiceList') {
-    origValFormula = `grist.ChoiceList.toString($${colId})`;
-  }
-
-  const toTypePure: string = gristTypes.extractTypeFromColType(newType);
-
-  // The args are used to construct the call to grist.TYPE.typeConvert(value, [params]).
-  // Optional parameters depend on the type; see sandbox/grist/usertypes.py
-  const args: string[] = [origValFormula];
-  switch (toTypePure) {
-    case 'Ref':
-    case 'RefList':
-    {
-      const table = gutil.removePrefix(newType, toTypePure + ":");
-      args.push(table || 'None');
-      const visibleColName = getVisibleColName(docModel, newVisibleCol);
-      if (visibleColName) {
-        args.push(q(visibleColName));
-      }
-      break;
-    }
-    case 'Date': {
-      args.push(q(newWidgetOptions.dateFormat));
-      break;
-    }
-    case 'DateTime': {
-      const timezone = gutil.removePrefix(newType, "DateTime:") || '';
-      const format = newWidgetOptions.dateFormat + ' ' + newWidgetOptions.timeFormat;
-      args.push(q(format), q(timezone));
-      break;
-    }
-  }
-  return `grist.${gristTypes.getGristType(toTypePure)}.typeConvert(${args.join(', ')})`;
-}
-
-function q(value: string): string {
-  return "'" + value.replace(/'/g, "\\'") + "'";
-}
-
 // Returns the name of the visibleCol given its rowId.
 function getVisibleColName(docModel: DocModel, visibleColRef: number): string|undefined {
  return visibleColRef ? docModel.columns.getRowModel(visibleColRef).colId() : undefined;
--- a/app/client/components/TypeTransform.ts
+++ b/app/client/components/TypeTransform.ts
@@ -10,13 +10,12 @@ import {ColumnTransform} from 'app/client/components/ColumnTransform';
 import {GristDoc} from 'app/client/components/GristDoc';
 import * as TypeConversion from 'app/client/components/TypeConversion';
 import {reportError} from 'app/client/models/errors';
-import * as modelUtil from 'app/client/models/modelUtil';
 import {cssButtonRow} from 'app/client/ui/RightPanel';
 import {basicButton, primaryButton} from 'app/client/ui2018/buttons';
 import {testId} from 'app/client/ui2018/cssVars';
 import {FieldBuilder} from 'app/client/widgets/FieldBuilder';
 import {NewAbstractWidget} from 'app/client/widgets/NewAbstractWidget';
-import {ColValues} from 'app/common/DocActions';
+import {ColValues, UserAction} from 'app/common/DocActions';
 import {Computed, dom, fromKo, Observable} from 'grainjs';
 import isEmpty = require('lodash/isEmpty');
 import pickBy = require('lodash/pickBy');
@@ -86,15 +85,6 @@ export class TypeTransform extends ColumnTransform {
    );
  }

-  protected async resetToDefaultFormula() {
-    if (!this.isFinalizing()) {
-      const toType = this.transformColumn.type.peek();
-      const formula = TypeConversion.getDefaultFormula(this.gristDoc.docModel, this.origColumn,
-        toType, this.field.visibleColRef(), this.field.widgetOptionsJson());
-      await modelUtil.setSaveValue(this.transformColumn.formula, formula);
-    }
-  }
-
  /**
   * Overrides parent method to initialize the transform column with guesses as to the particular
   * type and column options.
@@ -103,20 +93,55 @@ export class TypeTransform extends ColumnTransform {
  protected async addTransformColumn(toType: string) {
    const docModel = this.gristDoc.docModel;
    const colInfo = await TypeConversion.prepTransformColInfo(docModel, this.origColumn, this.origDisplayCol, toType);
-    const newColInfo = await this._tableData.sendTableAction(['AddColumn', 'gristHelper_Transform', colInfo]);
-    const tcol = docModel.columns.getRowModel(newColInfo.colRef);
-    await TypeConversion.setDisplayFormula(docModel, tcol);
-    return newColInfo.colRef;
+    const newColInfos = await this._tableData.sendTableActions([
+      ['AddColumn', 'gristHelper_Converted', {...colInfo, isFormula: false, formula: ''}],
+      ['AddColumn', 'gristHelper_Transform', colInfo],
+    ]);
+    const transformColRef = newColInfos[1].colRef;
+    this.transformColumn = docModel.columns.getRowModel(transformColRef);
+    await this.convertValues();
+    return transformColRef;
+  }
+
+  protected convertValuesActions(): UserAction[] {
+    const tableId = this._tableData.tableId;
+    const srcColId = this.origColumn.colId.peek();
+    const dstColId = "gristHelper_Converted";
+    const type = this.transformColumn.type.peek();
+    const widgetOptions = this.transformColumn.widgetOptions.peek();
+    const visibleColRef = this.transformColumn.visibleCol.peek();
+    return [['ConvertFromColumn', tableId, srcColId, dstColId, type, widgetOptions, visibleColRef]];
+  }
+
+  protected async convertValues() {
+    await Promise.all([
+      this.gristDoc.docData.sendActions(this.convertValuesActions()),
+      TypeConversion.setDisplayFormula(this.gristDoc.docModel, this.transformColumn),
+    ]);
+  }
+
+  protected executeActions(): UserAction[] {
+    return [...this.convertValuesActions(), ...super.executeActions()];
  }

  /**
   * Overrides parent method to subscribe to changes to the transform column.
   */
  protected postAddTransformColumn() {
-    // When a user-initiated change is saved to type or widgetOptions, update the formula.
-    this.autoDispose(this.transformColumn.type.subscribe(this.resetToDefaultFormula, this, "save"));
-    this.autoDispose(this.transformColumn.visibleCol.subscribe(this.resetToDefaultFormula, this, "save"));
-    this.autoDispose(this.field.widgetOptionsJson.subscribe(this.resetToDefaultFormula, this, "save"));
+    // When a user-initiated change is saved to type or widgetOptions, reconvert the values
+    // Need to subscribe to both 'change' and 'save' for type which can come from setting the type itself
+    // or e.g. a change to DateTime timezone.
+    this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "change"));
+    this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "save"));
+    this.autoDispose(this.transformColumn.visibleCol.subscribe(this.convertValues, this, "save"));
+    this.autoDispose(this.field.widgetOptionsJson.subscribe(this.convertValues, this, "save"));
+  }
+
+  /**
+   * Overrides parent method to delete extra column
+   */
+  protected cleanup() {
+    void this._tableData.sendTableAction(['RemoveColumn', 'gristHelper_Converted']);
  }

  /**
@@ -129,9 +154,10 @@ export class TypeTransform extends ColumnTransform {
    const tcol = this.transformColumn;
    const changedInfo = pickBy(colInfo, (val, key) =>
      (val !== tcol[key as keyof TypeConversion.ColInfo].peek()));
-    return Promise.all([
-      isEmpty(changedInfo) ? undefined : tcol.updateColValues(changedInfo as ColValues),
-      TypeConversion.setDisplayFormula(docModel, tcol, changedInfo.visibleCol)
-    ]);
+    if (!isEmpty(changedInfo)) {
+      // Update the transform column, particularly the type.
+      // This will trigger the subscription in postAddTransformColumn and lead to calling convertValues.
+      await tcol.updateColValues(changedInfo as ColValues);
+    }
  }
 }
--- a/app/client/models/DocModel.ts
+++ b/app/client/models/DocModel.ts
@@ -22,7 +22,7 @@ import {urlState} from 'app/client/models/gristUrlState';
 import * as MetaRowModel from 'app/client/models/MetaRowModel';
 import * as MetaTableModel from 'app/client/models/MetaTableModel';
 import * as rowset from 'app/client/models/rowset';
-import {isHiddenTable} from 'app/client/models/isHiddenTable';
+import {isHiddenTable} from 'app/common/isHiddenTable';
 import {schema, SchemaTypes} from 'app/common/schema';

 import {ACLRuleRec, createACLRuleRec} from 'app/client/models/entities/ACLRuleRec';
--- a/app/client/models/entities/ColumnRec.ts
+++ b/app/client/models/entities/ColumnRec.ts
@@ -2,8 +2,13 @@ import {KoArray} from 'app/client/lib/koArray';
 import {DocModel, IRowModel, recordSet, refRecord, TableRec, ViewFieldRec} from 'app/client/models/DocModel';
 import {jsonObservable, ObjObservable} from 'app/client/models/modelUtil';
 import * as gristTypes from 'app/common/gristTypes';
-import {getReferencedTableId, isFullReferencingType} from 'app/common/gristTypes';
-import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter';
+import {getReferencedTableId} from 'app/common/gristTypes';
+import {
+  BaseFormatter,
+  createFullFormatterRaw,
+  createVisibleColFormatterRaw,
+  FullFormatterArgs
+} from 'app/common/ValueFormatter';
 import * as ko from 'knockout';

 // Represents a column in a user-defined table.
@@ -124,38 +129,23 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void {

  // Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol
  // associated with this column. If no visible column available, return formatting for the column itself.
-  this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this, docModel));
+  this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'vcol'));

-  this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, this.visibleColFormatter()));
-}
-
-export function visibleColFormatterForRec(
-  rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel
-): BaseFormatter {
-  const vcol = rec.visibleColModel();
-  const documentSettings = docModel.docInfoRow.documentSettingsJson();
-  const type = colRec.type();
-  if (isFullReferencingType(type)) {
-    if (vcol.getRowId() === 0) {
-      // This column displays the Row ID, e.g. Table1[2]
-      // referencedTableId may actually be empty if the table is hidden
-      const referencedTableId: string = colRec.refTable()?.tableId() || "";
-      return createFormatter('Id', {tableId: referencedTableId}, documentSettings);
-    } else {
-      return createFormatter(vcol.type(), vcol.widgetOptionsJson(), documentSettings);
-    }
-  } else {
-    // For non-reference columns, there's no 'visible column' and we just return a regular formatter
-    return createFormatter(type, rec.widgetOptionsJson(), documentSettings);
-  }
+  this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'full'));
 }

 export function formatterForRec(
-  rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, visibleColFormatter: BaseFormatter
+  rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, kind: 'full' | 'vcol'
 ): BaseFormatter {
-  const type = colRec.type();
-  // Ref/RefList columns delegate most formatting to the visibleColFormatter
-  const widgetOpts = {...rec.widgetOptionsJson(), visibleColFormatter};
-  const documentSettings = docModel.docInfoRow.documentSettingsJson();
-  return createFormatter(type, widgetOpts, documentSettings);
+  const vcol = rec.visibleColModel();
+  const func = kind === 'full' ? createFullFormatterRaw : createVisibleColFormatterRaw;
+  const args: FullFormatterArgs = {
+    docData: docModel.docData,
+    type: colRec.type(),
+    widgetOpts: rec.widgetOptionsJson(),
+    visibleColType: vcol?.type(),
+    visibleColWidgetOpts: vcol?.widgetOptionsJson(),
+    docSettings: docModel.docInfoRow.documentSettingsJson(),
+  };
+  return func(args);
 }
--- a/app/client/models/entities/ViewFieldRec.ts
+++ b/app/client/models/entities/ViewFieldRec.ts
@@ -1,5 +1,5 @@
 import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel';
-import {formatterForRec, visibleColFormatterForRec} from 'app/client/models/entities/ColumnRec';
+import {formatterForRec} from 'app/client/models/entities/ColumnRec';
 import * as modelUtil from 'app/client/models/modelUtil';
 import * as UserType from 'app/client/widgets/UserType';
 import {DocumentSettings} from 'app/common/DocumentSettings';
@@ -172,13 +172,14 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void

  // Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol
  // associated with this field. If no visible column available, return formatting for the field itself.
-  this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this.column(), docModel));
+  this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'vcol'));

-  this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, this.visibleColFormatter()));
+  this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'full'));

  this.createValueParser = function() {
    const fieldRef = this.useColOptions.peek() ? undefined : this.id.peek();
-    return createParser(docModel.docData, this.colRef.peek(), fieldRef);
+    const parser = createParser(docModel.docData, this.colRef.peek(), fieldRef);
+    return parser.cleanParse.bind(parser);
  };

  // The widgetOptions to read and write: either the column's or the field's own.
--- a/app/client/ui/Pages.ts
+++ b/app/client/ui/Pages.ts
@@ -3,7 +3,7 @@ import { duplicatePage } from "app/client/components/duplicatePage";
 import { GristDoc } from "app/client/components/GristDoc";
 import { PageRec } from "app/client/models/DocModel";
 import { urlState } from "app/client/models/gristUrlState";
-import { isHiddenTable } from 'app/client/models/isHiddenTable';
+import { isHiddenTable } from 'app/common/isHiddenTable';
 import * as MetaTableModel from "app/client/models/MetaTableModel";
 import { find as findInTree, fromTableData, TreeItemRecord, TreeRecord,
         TreeTableData} from "app/client/models/TreeModel";
--- a/app/common/ValueConverter.ts
+++ b/app/common/ValueConverter.ts
@@ -0,0 +1,255 @@
+import {DocData} from 'app/common/DocData';
+import * as gristTypes from 'app/common/gristTypes';
+import {isList} from 'app/common/gristTypes';
+import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFormatter';
+import {
+  createParserOrFormatterArgumentsRaw,
+  createParserRaw,
+  ReferenceListParser,
+  ReferenceParser,
+  ValueParser
+} from 'app/common/ValueParser';
+import {CellValue, GristObjCode} from 'app/plugin/GristData';
+
+
+/**
+ * Base class for converting values from one type to another with the convert() method.
+ * Has a formatter for the source column
+ * and a parser for the destination column.
+ *
+ * The default convert() is for non-list destination types, so if the source value
+ * is a list it only converts nicely if the list contains exactly one element.
+ */
+export class ValueConverter {
+  constructor(public formatter: BaseFormatter, public parser: ValueParser) {
+  }
+
+  public convert(value: any): any {
+    if (isList(value)) {
+      if (value.length === 1) {
+        // Empty list: ['L']
+        return null;
+      } else if (value.length === 2) {
+        // Singleton list: ['L', value]
+        // Convert just that one value.
+        value = value[1];
+      } else {
+        // List with multiple values. Since we're converting to just one value,
+        // format the whole thing as text, which is an error for most types.
+        return this.formatter.formatAny(value);
+      }
+    }
+    return this.convertInner(value);
+  }
+
+  protected convertInner(value: any): any {
+    const formatted = this.formatter.formatAny(value);
+    return this.parser.cleanParse(formatted);
+  }
+}
+
+/**
+ * Base class for converting to a list type (Reference List or Choice List).
+ *
+ * Wraps single values in a list, and converts lists elementwise.
+ */
+class ListConverter extends ValueConverter {
+  // Don't parse strings like "Smith, John" which may look like lists but represent a single choice.
+  // TODO this works when the source is a Choice column, but not when it's a Reference to a Choice column.
+  //   But the guessed choices are also broken in that case.
+  private _choices: Set<string> = new Set((this.formatter.widgetOpts as any).choices || []);
+
+  public convert(value: any): any {
+    if (typeof value === "string" && !this._choices.has(value)) {
+      // Parse CSV/JSON
+      return this.parser.cleanParse(value);
+    }
+    const values = isList(value) ? value.slice(1) : [value];
+    if (!values.length || value == null) {
+      return null;
+    }
+    return this.handleValues(value, values.map(v => this.convertInner(v)));
+  }
+
+  protected handleValues(originalValue: any, values: any[]) {
+    return ['L', ...values];
+  }
+}
+
+class ChoiceListConverter extends ListConverter {
+  /**
+   * Convert each source value to a 'Choice'
+   */
+  protected convertInner(value: any): any {
+    return this.formatter.formatAny(value);
+  }
+}
+
+class ReferenceListConverter extends ListConverter {
+  private _innerConverter = new ReferenceConverter(
+    this.formatter,
+    new ReferenceParser("Ref", this.parser.widgetOpts, this.parser.docSettings),
+  );
+
+  constructor(public formatter: BaseFormatter, public parser: ReferenceListParser) {
+    super(formatter, parser);
+    // Prevent the parser from looking up reference values in the frontend.
+    // Leave it to the data engine which has a much more efficient algorithm for long lists of values.
+    delete parser.tableData;
+  }
+
+  public handleValues(originalValue: any, values: any[]): any {
+    const result = [];
+    let lookupColumn: string = "";
+    const raw = this.formatter.formatAny(originalValue);  // AltText if the reference lookup fails
+    for (const value of values) {
+      if (typeof value === "string") {
+        // Failed to parse one of the references, so return a raw string for the whole thing
+        return raw;
+      } else {
+        // value is a lookup tuple: ['l', value, options]
+        result.push(value[1]);
+        lookupColumn = value[2].column;
+      }
+    }
+    return ['l', result, {column: lookupColumn, raw}];
+  }
+
+  /**
+   * Convert each source value to a 'Reference'
+   */
+  protected convertInner(value: any): any {
+    return this._innerConverter.convert(value);
+  }
+}
+
+class ReferenceConverter extends ValueConverter {
+  private _innerConverter: ValueConverter = createConverter(this.formatter, this.parser.visibleColParser);
+
+  constructor(public formatter: BaseFormatter, public parser: ReferenceParser) {
+    super(formatter, parser);
+    // Prevent the parser from looking up reference values in the frontend.
+    // Leave it to the data engine which has a much more efficient algorithm for long lists of values.
+    delete parser.tableData;
+  }
+
+  protected convertInner(value: any): any {
+    // Convert to the type of the visible column.
+    const converted = this._innerConverter.convert(value);
+    return this.parser.lookup(converted, this.formatter.formatAny(value));
+  }
+}
+
+class NumericConverter extends ValueConverter {
+  protected convertInner(value: any): any {
+    if (typeof value === "boolean") {
+      return value ? 1 : 0;
+    }
+    return super.convertInner(value);
+  }
+}
+
+class DateConverter extends ValueConverter {
+  private _sourceType = gristTypes.extractInfoFromColType(this.formatter.type);
+
+  protected convertInner(value: any): any {
+    // When converting Date->DateTime, DateTime->Date, or between DateTime timezones,
+    // it's important to send an encoded Date/DateTime object rather than just a timestamp number
+    // so that the data engine knows what to do in do_convert, especially regarding timezones.
+    // If the source column is a Reference to a Date/DateTime then `value` is already
+    // an encoded object from the display column which has type Any.
+    value = gristTypes.reencodeAsAny(value, this._sourceType);
+    if (Array.isArray(value) && (
+      value[0] === GristObjCode.Date ||
+      value[0] === GristObjCode.DateTime
+    )) {
+      return value;
+    }
+    return super.convertInner(value);
+  }
+}
+
+export const valueConverterClasses: { [type: string]: typeof ValueConverter } = {
+  Date: DateConverter,
+  DateTime: DateConverter,
+  ChoiceList: ChoiceListConverter,
+  Ref: ReferenceConverter,
+  RefList: ReferenceListConverter,
+  Numeric: NumericConverter,
+  Int: NumericConverter,
+};
+
+export function createConverter(formatter: BaseFormatter, parser: ValueParser) {
+  const cls = valueConverterClasses[gristTypes.extractTypeFromColType(parser.type)] || ValueConverter;
+  return new cls(formatter, parser);
+}
+
+/**
+ * Used by the ConvertFromColumn user action in the data engine.
+ * The higher order function separates docData (passed by ActiveDoc)
+ * from the arguments passed to call_external in Python.
+ */
+export function convertFromColumn(docData: DocData) {
+  return function(
+    sourceColRef: number,
+    type: string,
+    widgetOpts: string,
+    visibleColRef: number,
+    values: ReadonlyArray<CellValue>,
+    displayColValues?: ReadonlyArray<CellValue>,
+  ): CellValue[] {
+    const formatter = createFullFormatterFromDocData(docData, sourceColRef);
+    const parser = createParserRaw(
+      ...createParserOrFormatterArgumentsRaw(docData, type, widgetOpts, visibleColRef)
+    );
+    const converter = createConverter(formatter, parser);
+    return convertValues(converter, values, displayColValues || values);
+  };
+}
+
+export function convertValues(
+  converter: ValueConverter,
+  // Raw values from the actual column, e.g. row IDs for reference columns
+  values: ReadonlyArray<CellValue>,
+  // Values from the display column, which is the same as the raw values for non-referencing columns.
+  // In almost all cases these are the values that actually matter and get converted.
+  displayColValues: ReadonlyArray<CellValue>,
+): CellValue[] {
+  // Converting Ref <-> RefList without changing the target table is a special case - see prepTransformColInfo.
+  // In this case we deal with the actual row IDs stored in the real column,
+  // whereas in all other cases we use display column values.
+  const sourceType = gristTypes.extractInfoFromColType(converter.formatter.type);
+  const targetType = gristTypes.extractInfoFromColType(converter.parser.type);
+  const refToRefList = (
+    sourceType.type === "Ref" &&
+    targetType.type === "RefList" &&
+    sourceType.tableId === targetType.tableId
+  );
+  const refListToRef = (
+    sourceType.type === "RefList" &&
+    targetType.type === "Ref" &&
+    sourceType.tableId === targetType.tableId
+  );
+
+  return displayColValues.map((displayVal, i) => {
+    const actualValue = values[i];
+
+    if (refToRefList && typeof actualValue === "number") {
+      if (actualValue === 0) {
+        return null;
+      } else {
+        return ["L", actualValue];
+      }
+    } else if (refListToRef && isList(actualValue)) {
+      if (actualValue.length === 1) {
+        // Empty list: ['L']
+        return 0;
+      } else if (actualValue.length === 2) {
+        // Singleton list: ['L', rowId]
+        return actualValue[1];
+      }
+    }
+
+    return converter.convert(displayVal);
+  });
+}
--- a/app/common/ValueFormatter.ts
+++ b/app/common/ValueFormatter.ts
@@ -2,11 +2,14 @@

 import {csvEncodeRow} from 'app/common/csvFormat';
 import {CellValue} from 'app/common/DocActions';
+import {DocData} from 'app/common/DocData';
 import {DocumentSettings} from 'app/common/DocumentSettings';
-import {getReferencedTableId, isList} from 'app/common/gristTypes';
 import * as gristTypes from 'app/common/gristTypes';
+import {getReferencedTableId, isList} from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
+import {isHiddenTable} from 'app/common/isHiddenTable';
 import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
+import {createParserOrFormatterArguments, ReferenceParsingOptions} from 'app/common/ValueParser';
 import {GristObjCode} from 'app/plugin/GristData';
 import {decodeObject, GristDateTime} from 'app/plugin/objtypes';
 import * as moment from 'moment-timezone';
@@ -280,3 +283,70 @@ export function createFormatter(type: string, widgetOpts: FormatOptions, docSett
  const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter;
  return new ctor(type, widgetOpts, docSettings);
 }
+
+export interface FullFormatterArgs {
+  docData: DocData;
+  type: string;
+  widgetOpts: FormatOptions;
+  visibleColType: string;
+  visibleColWidgetOpts: FormatOptions;
+  docSettings: DocumentSettings;
+}
+
+/**
+ * Returns a constructor
+ * with a format function that can properly convert a value passed to it into the
+ * right format for that column.
+ *
+ * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
+ * instead of the table column.
+ */
+export function createFullFormatterFromDocData(
+  docData: DocData,
+  colRef: number,
+  fieldRef?: number,
+): BaseFormatter {
+  const [type, widgetOpts, docSettings] = createParserOrFormatterArguments(docData, colRef, fieldRef);
+  const {visibleColType, visibleColWidgetOpts} = widgetOpts as ReferenceParsingOptions;
+  return createFullFormatterRaw({
+    docData,
+    type,
+    widgetOpts,
+    visibleColType,
+    visibleColWidgetOpts,
+    docSettings,
+  });
+}
+
+export function createFullFormatterRaw(args: FullFormatterArgs) {
+  const {type, widgetOpts, docSettings} = args;
+  const visibleColFormatter = createVisibleColFormatterRaw(args);
+  return createFormatter(type, {...widgetOpts, visibleColFormatter}, docSettings);
+}
+
+export function createVisibleColFormatterRaw(
+  {
+    docData,
+    docSettings,
+    type,
+    visibleColType,
+    visibleColWidgetOpts,
+    widgetOpts
+  }: FullFormatterArgs
+): BaseFormatter {
+  let referencedTableId = gristTypes.getReferencedTableId(type);
+  if (!referencedTableId) {
+    return createFormatter(type, widgetOpts, docSettings);
+  } else if (visibleColType) {
+    return createFormatter(visibleColType, visibleColWidgetOpts, docSettings);
+  } else {
+    // This column displays the Row ID, e.g. Table1[2]
+    // Make referencedTableId empty if the table is hidden
+    const tablesData = docData.getMetaTable("_grist_Tables");
+    const tableRef = tablesData.findRow("tableId", referencedTableId);
+    if (isHiddenTable(tablesData, tableRef)) {
+      referencedTableId = "";
+    }
+    return createFormatter('Id', {tableId: referencedTableId}, docSettings);
+  }
+}
--- a/app/common/ValueParser.ts
+++ b/app/common/ValueParser.ts
@@ -33,6 +33,9 @@ export class ValueParser {

 }

+class IdentityParser extends ValueParser {
+}
+
 /**
 * Same as basic Value parser, but will return null if a value is an empty string.
 */
@@ -117,7 +120,7 @@ class ChoiceListParser extends ValueParser {
 * stored on the field. These have to be specially derived
 * for referencing columns. See createParser.
 */
-interface ReferenceParsingOptions {
+export interface ReferenceParsingOptions {
  visibleColId: string;
  visibleColType: string;
  visibleColWidgetOpts: FormatOptions;
@@ -129,18 +132,22 @@ interface ReferenceParsingOptions {

 export class ReferenceParser extends ValueParser {
  public widgetOpts: ReferenceParsingOptions;
-
-  protected _visibleColId = this.widgetOpts.visibleColId;
-  protected _tableData = this.widgetOpts.tableData;
-  protected _visibleColParser = createParserRaw(
+  public tableData = this.widgetOpts.tableData;
+  public visibleColParser = createParserRaw(
    this.widgetOpts.visibleColType,
    this.widgetOpts.visibleColWidgetOpts,
    this.docSettings,
  );

+  protected _visibleColId = this.widgetOpts.visibleColId;
+
  public parse(raw: string): any {
-    let value = this._visibleColParser(raw);
-    if (!value || !raw) {
+    const value = this.visibleColParser.cleanParse(raw);
+    return this.lookup(value, raw);
+  }
+
+  public lookup(value: any, raw: string): any {
+    if (value == null || value === "" || !raw) {
      return 0;  // default value for a reference column
    }

@@ -154,7 +161,7 @@ export class ReferenceParser extends ValueParser {
      }
    }

-    if (!this._tableData?.isLoaded) {
+    if (!this.tableData?.isLoaded) {
      const options: { column: string, raw?: string } = {column: this._visibleColId};
      if (value !== raw) {
        options.raw = raw;
@@ -162,7 +169,7 @@ export class ReferenceParser extends ValueParser {
      return ['l', value, options];
    }

-    return this._tableData.findMatchingRowId({[this._visibleColId]: value}) || raw;
+    return this.tableData.findMatchingRowId({[this._visibleColId]: value}) || raw;
  }
 }

@@ -178,7 +185,7 @@ export class ReferenceListParser extends ReferenceParser {
      // csvDecodeRow should never raise an exception
      values = csvDecodeRow(raw);
    }
-    values = values.map(v => typeof v === "string" ? this._visibleColParser(v) : encodeObject(v));
+    values = values.map(v => typeof v === "string" ? this.visibleColParser.cleanParse(v) : encodeObject(v));

    if (!values.length || !raw) {
      return null;  // null is the default value for a reference list column
@@ -194,7 +201,7 @@ export class ReferenceListParser extends ReferenceParser {
      }
    }

-    if (!this._tableData?.isLoaded) {
+    if (!this.tableData?.isLoaded) {
      const options: { column: string, raw?: string } = {column: this._visibleColId};
      if (!(values.length === 1 && values[0] === raw)) {
        options.raw = raw;
@@ -204,7 +211,7 @@ export class ReferenceListParser extends ReferenceParser {

    const rowIds: number[] = [];
    for (const value of values) {
-      const rowId = this._tableData.findMatchingRowId({[this._visibleColId]: value});
+      const rowId = this.tableData.findMatchingRowId({[this._visibleColId]: value});
      if (rowId) {
        rowIds.push(rowId);
      } else {
@@ -228,27 +235,21 @@ export const valueParserClasses: { [type: string]: typeof ValueParser } = {
  RefList: ReferenceListParser,
 };

-const identity = (value: string) => value;
-
 /**
- * Returns a function which can parse strings into values appropriate for
+ * Returns a ValueParser which can parse strings into values appropriate for
 * a specific widget field or table column.
 * widgetOpts is usually the field/column's widgetOptions JSON
 * but referencing columns need more than that, see ReferenceParsingOptions above.
 */
 export function createParserRaw(
  type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings
-): (value: string) => any {
-  const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)];
-  if (cls) {
-    const parser = new cls(type, widgetOpts, docSettings);
-    return parser.cleanParse.bind(parser);
-  }
-  return identity;
+): ValueParser {
+  const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)] || IdentityParser;
+  return new cls(type, widgetOpts, docSettings);
 }

 /**
- * Returns a function which can parse strings into values appropriate for
+ * Returns a ValueParser which can parse strings into values appropriate for
 * a specific widget field or table column.
 *
 * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
@@ -258,23 +259,46 @@ export function createParser(
  docData: DocData,
  colRef: number,
  fieldRef?: number,
-): (value: string) => any {
+): ValueParser {
+  return createParserRaw(...createParserOrFormatterArguments(docData, colRef, fieldRef));
+}
+
+/**
+ * Returns arguments suitable for createParserRaw or createFormatter. Only for internal use.
+ *
+ * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
+ * instead of the table column.
+ */
+export function createParserOrFormatterArguments(
+  docData: DocData,
+  colRef: number,
+  fieldRef?: number,
+): [string, object, DocumentSettings] {
  const columnsTable = docData.getMetaTable('_grist_Tables_column');
  const fieldsTable = docData.getMetaTable('_grist_Views_section_field');
-  const docInfoTable = docData.getMetaTable('_grist_DocInfo');

  const col = columnsTable.getRecord(colRef)!;
-
  let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col;
  if (fieldRef) {
    fieldOrCol = fieldsTable.getRecord(fieldRef) || col;
  }

-  const widgetOpts = safeJsonParse(fieldOrCol.widgetOptions, {});
+  return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol);
+}
+
+export function createParserOrFormatterArgumentsRaw(
+  docData: DocData,
+  type: string,
+  widgetOptions: string,
+  visibleColRef: number,
+): [string, object, DocumentSettings] {
+  const columnsTable = docData.getMetaTable('_grist_Tables_column');
+  const docInfoTable = docData.getMetaTable('_grist_DocInfo');
+
+  const widgetOpts = safeJsonParse(widgetOptions, {});

-  const type = col.type;
  if (isFullReferencingType(type)) {
-    const vcol = columnsTable.getRecord(fieldOrCol.visibleCol);
+    const vcol = columnsTable.getRecord(visibleColRef);
    widgetOpts.visibleColId = vcol?.colId || 'id';
    widgetOpts.visibleColType = vcol?.type;
    widgetOpts.visibleColWidgetOpts = safeJsonParse(vcol?.widgetOptions || '', {});
@@ -284,7 +308,7 @@ export function createParser(
  const docInfo = docInfoTable.getRecord(1);
  const docSettings = safeJsonParse(docInfo!.documentSettings, {}) as DocumentSettings;

-  return createParserRaw(type, widgetOpts, docSettings);
+  return [type, widgetOpts, docSettings];
 }

 /**
@@ -311,12 +335,12 @@ function parseColValues<T extends ColValues | BulkColValues>(
    const parser = createParser(docData, colRef);

    // Optimisation: If there's no special parser for this column type, do nothing
-    if (parser === identity) {
+    if (parser instanceof IdentityParser) {
      return values;
    }

    function parseIfString(val: any) {
-      return typeof val === "string" ? parser(val) : val;
+      return typeof val === "string" ? parser.cleanParse(val) : val;
    }

    if (bulk) {
--- a/app/client/models/isHiddenTable.ts
+++ b/app/client/models/isHiddenTable.ts
@@ -1,11 +1,11 @@
-import {RowId} from 'app/client/models/rowset';
-import {TableData} from 'app/client/models/TableData';
+import {UIRowId} from 'app/common/UIRowId';
+import {TableData} from "./TableData";

 /**
 * Return whether a table identified by the rowId of its metadata record, should normally be
 * hidden from the user (e.g. as an option in the page-widget picker).
 */
-export function isHiddenTable(tablesData: TableData, tableRef: RowId): boolean {
+export function isHiddenTable(tablesData: TableData, tableRef: UIRowId): boolean {
  const tableId = tablesData.getValue(tableRef, 'tableId') as string|undefined;
  return tablesData.getValue(tableRef, 'summarySourceTable') !== 0 ||
    Boolean(tableId?.startsWith('GristHidden'));
--- a/app/server/lib/ActiveDoc.ts
+++ b/app/server/lib/ActiveDoc.ts
@@ -41,6 +41,7 @@ import {schema, SCHEMA_VERSION} from 'app/common/schema';
 import {MetaRowRecord} from 'app/common/TableData';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
+import {convertFromColumn} from 'app/common/ValueConverter';
 import {parseUserAction} from 'app/common/ValueParser';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
 import {GristDocAPI} from 'app/plugin/GristAPI';
@@ -1717,6 +1718,12 @@ export class ActiveDoc extends EventEmitter {
      logTimes: true,
      logMeta: {docId: this._docName},
      preferredPythonVersion,
+      sandboxOptions: {
+        exports: {
+          convertFromColumn: (...args: Parameters<ReturnType<typeof convertFromColumn>>) =>
+            convertFromColumn(this.docData!)(...args)
+        }
+      },
    });
  }
 }
--- a/app/server/lib/GranularAccess.ts
+++ b/app/server/lib/GranularAccess.ts
@@ -87,6 +87,7 @@ const SPECIAL_ACTIONS = new Set(['InitNewDoc',
                                 'TransformAndFinishImport',
                                 'AddView',
                                 'CopyFromColumn',
+                                 'ConvertFromColumn',
                                 'AddHiddenColumn',
                                ]);

--- a/app/server/lib/ISandbox.ts
+++ b/app/server/lib/ISandbox.ts
@@ -1,4 +1,5 @@
 import * as log from 'app/server/lib/log';
+import {ISandboxOptions} from 'app/server/lib/NSandbox';

 /**
 * Starting to whittle down the options used when creating a sandbox, to leave more
@@ -17,6 +18,8 @@ export interface ISandboxCreationOptions {
  importMount?: string;  // if defined, make this path available read-only as "/importdir"

  preferredPythonVersion?: '2' | '3';
+
+  sandboxOptions?: Partial<ISandboxOptions>;
 }

 export interface ISandbox {
--- a/app/server/lib/NSandbox.ts
+++ b/app/server/lib/NSandbox.ts
@@ -33,7 +33,7 @@ type SandboxMethod = (...args: any[]) => any;
 * started by setting `useGristEntrypoint` (the only exception is
 * in tests) which runs grist/main.py.
 */
-interface ISandboxOptions {
+export interface ISandboxOptions {
  command?: string;       // External program or container to call to run the sandbox.
  args: string[];         // The arguments to pass to the python process.

@@ -404,6 +404,7 @@ export class NSandboxCreator implements ISandboxCreator {
      preferredPythonVersion: this._preferredPythonVersion || options.preferredPythonVersion,
      useGristEntrypoint: true,
      importDir: options.importMount,
+      ...options.sandboxOptions,
    };
    return new NSandbox(translatedOptions, spawners[this._flavor]);
  }