(core) Fix imports into reference columns, and support two ways to import Numeric as a reference.

Summary: - When importing into a Ref column, use lookupOne() formula for correct previews. - When selecting columns to import into a Ref column, now a Numeric column like 'Order' will produce two options: "Order" and "Order (as row ID)". - Fixes exports to correct the formatting of visible columns. This addresses multiple bugs: 1. Formatting wasn't used, e.g. a Ref showing a custom-formatted date was still presented as YYYY-MM-DD in CSVs. 2. Ref showing a Numeric column was formatted as if a row ID (e.g. `Table1[1.5]`), which is very wrong. - If importing into a table that doesn't have a primary view, don't switch page after import. Refactorings: - Generalize GenImporterView to be usable in more cases; removed near-duplicated logic from node side - Some other refactoring in importing code. - Fix field/column option selection in ValueParser - Add NUM() helper to turn integer-valued floats into ints, useful for "as row ID" lookups. Test Plan: Added test cases for imports into reference columns, updated Exports test fixtures. Reviewers: georgegevoian Reviewed By: georgegevoian Differential Revision: https://phab.getgrist.com/D3875
2026-03-02 04:09:24 +00:00 · 2023-04-25 17:11:25 -04:00
parent 7a12a8ef28
commit 65013331a3
17 changed files with 290 additions and 339 deletions
--- a/app/client/components/Importer.ts
+++ b/app/client/components/Importer.ts
@@ -19,7 +19,8 @@ import {openFilePicker} from 'app/client/ui/FileDialog';
 import {bigBasicButton, bigPrimaryButton} from 'app/client/ui2018/buttons';
 import {testId, theme, vars} from 'app/client/ui2018/cssVars';
 import {icon} from 'app/client/ui2018/icons';
-import {IOptionFull, linkSelect, menu, menuDivider, menuItem, multiSelect} from 'app/client/ui2018/menus';
+import {IOptionFull, linkSelect, menu,
+        menuDivider, menuItem, multiSelect} from 'app/client/ui2018/menus';
 import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals';
 import {loadingSpinner} from 'app/client/ui2018/loaders';
 import {openFormulaEditor} from 'app/client/widgets/FormulaEditor';
@@ -188,30 +189,29 @@ export class Importer extends DisposableWithEvents {
    ...use(this._gristDoc.docModel.visibleTableIds.getObservable()).map((id) => ({value: id, label: id})),
  ]);

-  // Source column labels for the selected import source, keyed by column id.
-  private _sourceColLabelsById = Computed.create(this, this._sourceInfoSelected, (use, info) => {
-    if (!info || use(info.sourceSection._isDeleted)) { return null; }
-
-    const fields = use(use(info.sourceSection.viewFields).getObservable());
-    return new Map(fields.map(f => [use(use(f.column).colId), use(use(f.column).label)]));
+  // List of transform fields, i.e. those formula fields of the transform section whose values
+  // will be used to populate the destination columns.
+  private _transformFields: Computed<ViewFieldRec[]|null> = Computed.create(
+      this, this._sourceInfoSelected, (use, info) => {
+    const section = info && use(info.transformSection);
+    if (!section || use(section._isDeleted)) { return null; }
+    return use(use(section.viewFields).getObservable());
  });

-  // Transform section columns of the selected source.
-  private _transformSectionCols = Computed.create(this, this._sourceInfoSelected, (use, info) => {
-    if (!info) { return null; }
-
-    const transformSection = use(info.transformSection);
-    if (!transformSection || use(transformSection._isDeleted)) { return null; }
-
-    const fields = use(use(transformSection.viewFields).getObservable());
-    return fields.map(f => use(f.column));
+  // Prepare a Map, mapping of colRef of each transform column to the set of options to offer in
+  // the dropdown. The options are represented as a Map too, mapping formula to label.
+  private _transformColImportOptions: Computed<Map<number, Map<string, string>>> = Computed.create(
+      this, this._transformFields, this._sourceInfoSelected, (use, fields, info) => {
+    if (!fields || !info) { return new Map(); }
+    return new Map(fields.map(f =>
+      [use(f.colRef), this._makeImportOptionsForCol(use(f.column), info)]));
  });

-  // List of destination fields that aren't mapped to a source column.
-  private _unmatchedFields = Computed.create(this, this._transformSectionCols, (use, cols) => {
-    if (!cols) { return null; }
-
-    return cols.filter(c => use(c.formula).trim() === '').map(c => c.label());
+  // List of labels of destination columns that aren't mapped to a source column, i.e. transform
+  // columns with empty formulas.
+  private _unmatchedFields: Computed<string[]|undefined> = Computed.create(
+      this, this._transformFields, (use, fields) => {
+    return fields?.filter(f => (use(use(f.column).formula).trim() === '')).map(f => use(f.label));
  });

  // null tells to use the built-in file picker.
@@ -299,9 +299,9 @@ export class Importer extends DisposableWithEvents {
    sourceInfo.transformSection.set(null);

    const genImporterViewPromise = this._gristDoc.docData.sendAction(
-      ['GenImporterView', sourceInfo.hiddenTableId, sourceInfo.destTableId.get(), null]);
+      ['GenImporterView', sourceInfo.hiddenTableId, sourceInfo.destTableId.get(), null, null]);
    sourceInfo.lastGenImporterViewPromise = genImporterViewPromise;
-    const transformSectionRef = await genImporterViewPromise;
+    const transformSectionRef = (await genImporterViewPromise).viewSectionRef;

    // If the request is superseded by a newer request, or the Importer is disposed, do nothing.
    if (this.isDisposed() || sourceInfo.lastGenImporterViewPromise !== genImporterViewPromise) {
@@ -444,7 +444,11 @@ export class Importer extends DisposableWithEvents {

    if (importResult.tables[0]?.hiddenTableId) {
      const tableRowModel = this._gristDoc.docModel.dataTables[importResult.tables[0].hiddenTableId].tableMetaRow;
-      await this._gristDoc.openDocPage(tableRowModel.primaryViewId());
+      const primaryViewId = tableRowModel.primaryViewId();
+      if (primaryViewId) {
+        // Switch page if there is a sensible one to switch to.
+        await this._gristDoc.openDocPage(primaryViewId);
+      }
    }
    this._screen.close();
    this.dispose();
@@ -685,49 +689,14 @@ export class Importer extends DisposableWithEvents {
                          ),
                          cssDestinationFieldSettings(
                            icon('Dots'),
-                            menu(
-                              () => {
-                                const sourceColId = field.origCol().id();
-                                const sourceColIdsAndLabels = [...this._sourceColLabelsById.get()!.entries()];
-                                return [
-                                  menuItem(
-                                    async () => {
-                                      await this._gristDoc.clearColumns([sourceColId], {keepType: true});
-                                      await this._updateImportDiff(info);
-                                    },
-                                    'Skip',
-                                    testId('importer-column-match-menu-item')
-                                  ),
-                                  menuDivider(),
-                                  ...sourceColIdsAndLabels.map(([id, label]) =>
-                                    menuItem(
-                                      async () => {
-                                        await this._setColumnFormula(sourceColId, '$' + id);
-                                        await this._updateImportDiff(info);
-                                      },
-                                      label,
-                                      testId('importer-column-match-menu-item')
-                                    ),
-                                  ),
-                                  testId('importer-column-match-menu'),
-                                ];
-                              },
+                            menu(() => this._makeImportOptionsMenu(field.origCol.peek(), info),
                              { placement: 'right-start' },
                            ),
                            testId('importer-column-match-destination-settings')
                          ),
                          testId('importer-column-match-destination')
                        ),
-                        dom.domComputed(use => dom.create(
-                          this._buildColMappingFormula.bind(this),
-                          use(field.column),
-                          (elem: Element) => this._activateFormulaEditor(
-                            elem,
-                            field,
-                            () => this._updateImportDiff(info),
-                          ),
-                          'Skip'
-                        )),
+                        dom.create(owner => this._buildColMappingFormula(owner, field, info)),
                        testId('importer-column-match-source-destination'),
                      )
                    )),
@@ -776,6 +745,52 @@ export class Importer extends DisposableWithEvents {
    this._screen.render(content, {fullscreen: true});
  }

+  private _makeImportOptionsForCol(transformCol: ColumnRec, info: SourceInfo) {
+    const options = new Map<string, string>();  // Maps formula to label.
+    const importedFields = info.sourceSection.viewFields.peek().peek();
+
+    // Reference columns are populated using lookup formulas, so figure out now if this is a
+    // reference column, and if so, its destination table and the lookup column ID.
+    const refTable = transformCol.refTable.peek();
+    const refTableId = refTable ? refTable.tableId.peek() : undefined;
+    const visibleColId = transformCol.visibleColModel.peek().colId.peek();
+    const isRefDest = Boolean(info.destTableId.get() && transformCol.pureType.peek() === 'Ref');
+
+    for (const f of importedFields) {
+      const importedCol = f.column.peek();
+      const colId = importedCol.colId.peek();
+      const colLabel = importedCol.label.peek();
+      if (isRefDest && visibleColId) {
+        const formula = `${refTableId}.lookupOne(${visibleColId}=$${colId}) or ($${colId} and str($${colId}))`;
+        options.set(formula, colLabel);
+      } else {
+        options.set(`$${colId}`, colLabel);
+      }
+      if (isRefDest && ['Numeric', 'Int'].includes(importedCol.type.peek())) {
+        options.set(`${refTableId}.lookupOne(id=NUM($${colId})) or ($${colId} and str(NUM($${colId})))`,
+          `${colLabel} (as row ID)`);
+      }
+    }
+    return options;
+  }
+
+  private _makeImportOptionsMenu(transformCol: ColumnRec, info: SourceInfo) {
+    const transformColRef = transformCol.id();
+    const options = this._transformColImportOptions.get().get(transformCol.getRowId());
+    return [
+      menuItem(() => this._setColumnFormula(transformColRef, null, info),
+        'Skip',
+        testId('importer-column-match-menu-item')),
+      menuDivider(),
+      ...Array.from(options || [], ([formula, label]) =>
+        menuItem(() => this._setColumnFormula(transformColRef, formula, info),
+          label,
+          testId('importer-column-match-menu-item'))
+      ),
+      testId('importer-column-match-menu'),
+    ];
+  }
+
  private _addFocusLayer(container: HTMLElement) {
    dom.autoDisposeElem(container, new FocusLayer({
      defaultFocusElem: container,
@@ -787,10 +802,14 @@ export class Importer extends DisposableWithEvents {
  /**
   * Updates the formula on column `colRef` to `formula`.
   */
-  private async _setColumnFormula(colRef: number, formula: string): Promise<void> {
-    return this._gristDoc.docModel.columns.sendTableAction(
-      ['UpdateRecord', colRef, { formula, isFormula: true }]
-    );
+  private async _setColumnFormula(transformColRef: number, formula: string|null, info: SourceInfo) {
+    if (formula === null) {
+      await this._gristDoc.clearColumns([transformColRef], {keepType: true});
+    } else {
+      await this._gristDoc.docModel.columns.sendTableAction(
+        ['UpdateRecord', transformColRef, { formula, isFormula: true }]);
+    }
+    await this._updateImportDiff(info);
  }

  /**
@@ -841,26 +860,20 @@ export class Importer extends DisposableWithEvents {
   * in the column mapping section of Importer. On click, opens
   * an editor for the formula for `column`.
   */
-  private _buildColMappingFormula(_owner: MultiHolder, column: ColumnRec, buildEditor: (e: Element) => void,
-                                  placeholder: string) {
-    const formatFormula = (formula: string) => {
-      const sourceColLabels = this._sourceColLabelsById.get();
-      if (!sourceColLabels) { return formula; }
+  private _buildColMappingFormula(owner: MultiHolder, field: ViewFieldRec, info: SourceInfo) {
+    const displayFormula = Computed.create(owner, use => {
+      const column = use(field.column);
+      const formula = use(column.formula);
+      const importOptions = use(this._transformColImportOptions).get(column.getRowId());
+      return importOptions?.get(formula) ?? formula;
+    });

-      formula = formula.trim();
-      if (formula.startsWith('$') && sourceColLabels.has(formula.slice(1))) {
-        // For simple formulas that only reference a source column id, show the source column label.
-        return sourceColLabels.get(formula.slice(1))!;
-      }
-
-      return formula;
-    };
-
-    return cssFieldFormula(use => formatFormula(use(column.formula)),
-      {gristTheme: this._gristDoc.currentTheme, placeholder, maxLines: 1},
+    return cssFieldFormula(displayFormula,
+      {gristTheme: this._gristDoc.currentTheme, placeholder: 'Skip', maxLines: 1},
      dom.cls('disabled'),
      {tabIndex: '-1'},
-      dom.on('focus', (_ev, elem) => buildEditor(elem)),
+      dom.on('focus', (_ev, elem) =>
+        this._activateFormulaEditor(elem, field, () => this._updateImportDiff(info))),
      testId('importer-column-match-formula'),
    );
  }
--- a/app/common/ValueFormatter.ts
+++ b/app/common/ValueFormatter.ts
@@ -70,7 +70,7 @@ export type IsRightTypeFunc = (value: CellValue) => boolean;
 export class BaseFormatter {
  protected isRightType: IsRightTypeFunc;

-  constructor(public type: string, public widgetOpts: object, public docSettings: DocumentSettings) {
+  constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) {
    this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) ||
      gristTypes.isRightType('Any')!;
  }
@@ -144,7 +144,7 @@ export interface DateFormatOptions {
 }

 class DateFormatter extends BaseFormatter {
-  private _dateTimeFormat: string;
+  protected _dateTimeFormat: string;
  private _timezone: string;

  constructor(type: string, widgetOpts: DateFormatOptions, docSettings: DocumentSettings, timezone: string = 'UTC') {
@@ -194,9 +194,11 @@ export interface DateTimeFormatOptions extends DateFormatOptions {
 class DateTimeFormatter extends DateFormatter {
  constructor(type: string, widgetOpts: DateTimeFormatOptions, docSettings: DocumentSettings) {
    const timezone = gutil.removePrefix(type, "DateTime:") || '';
+    // Pass up the original widgetOpts. It's helpful to have them available; e.g. ExcelFormatter
+    // takes options from an initialized ValueFormatter.
+    super(type, widgetOpts, docSettings, timezone);
    const timeFormat = widgetOpts.timeFormat === undefined ? 'h:mma' : widgetOpts.timeFormat;
-    const dateFormat = (widgetOpts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
-    super(type, {dateFormat}, docSettings, timezone);
+    this._dateTimeFormat = (widgetOpts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
  }
 }

--- a/app/common/ValueParser.ts
+++ b/app/common/ValueParser.ts
@@ -268,7 +268,8 @@ export function createParserOrFormatterArguments(
  const col = columnsTable.getRecord(colRef)!;
  let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col;
  if (fieldRef) {
-    fieldOrCol = fieldsTable.getRecord(fieldRef) || col;
+    const field = fieldsTable.getRecord(fieldRef);
+    fieldOrCol = field?.widgetOptions ? field : col;
  }

  return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol);
--- a/app/server/lib/ActiveDocImport.ts
+++ b/app/server/lib/ActiveDocImport.ts
@@ -5,11 +5,11 @@ import * as _ from 'underscore';

 import {ColumnDelta, createEmptyActionSummary} from 'app/common/ActionSummary';
 import {ApplyUAResult, DataSourceTransformed, ImportOptions, ImportResult, ImportTableResult,
-        MergeOptions, MergeOptionsMap, MergeStrategy, SKIP_TABLE, TransformColumn,
+        MergeOptions, MergeOptionsMap, MergeStrategy, SKIP_TABLE,
        TransformRule,
        TransformRuleMap} from 'app/common/ActiveDocAPI';
 import {ApiError} from 'app/common/ApiError';
-import {BulkColValues, CellValue, fromTableDataAction, TableRecordValue, UserAction} from 'app/common/DocActions';
+import {BulkColValues, CellValue, fromTableDataAction, UserAction} from 'app/common/DocActions';
 import * as gutil from 'app/common/gutil';
 import {DocStateComparison} from 'app/common/UserAPI';
 import {guessColInfoForImports} from 'app/common/ValueGuesser';
@@ -339,9 +339,9 @@ export class ActiveDocImport {
      if (isHidden) {
        // Generate formula columns, view sections, etc
        const results: ApplyUAResult = await this._activeDoc.applyUserActions(docSession,
-          [['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null]]);
+          [['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null, null]]);

-        transformSectionRef = results.retValues[0];
+        transformSectionRef = results.retValues[0].viewSectionRef;
        createdTableId = hiddenTableId;

      } else {
@@ -391,36 +391,21 @@ export class ActiveDocImport {
   * the source and destination table.
   * @returns {string} The table id of the new or updated destination table.
   */
-  private async _transformAndFinishImport(docSession: OptDocSession,
-                                          hiddenTableId: string, destTableId: string,
-                                          intoNewTable: boolean, transformRule: TransformRule|null,
-                                          mergeOptions: MergeOptions|null): Promise<string> {
+  private async _transformAndFinishImport(
+    docSession: OptDocSession,
+    hiddenTableId: string, destTableId: string,
+    intoNewTable: boolean, transformRule: TransformRule|null,
+    mergeOptions: MergeOptions|null
+  ): Promise<string> {
    log.info("ActiveDocImport._transformAndFinishImport(%s, %s, %s, %s, %s)",
      hiddenTableId, destTableId, intoNewTable, transformRule, mergeOptions);
-    const srcCols = await this._activeDoc.getTableCols(docSession, hiddenTableId);

-    // Use a default transform rule if one was not provided by the client.
-    if (!transformRule) {
-      const transformDest = intoNewTable ? null : destTableId;
-      transformRule = await this._makeDefaultTransformRule(docSession, srcCols, transformDest);
-    }
-
-    // Transform rules from client may have prefixed column ids, so we need to strip them.
-    stripRulePrefixes(transformRule);
-
-    if (intoNewTable) {
-      // Transform rules for new tables don't have filled in destination column ids.
-      const result = await this._activeDoc.applyUserActions(docSession, [['FillTransformRuleColIds', transformRule]]);
-      transformRule = result.retValues[0] as TransformRule;
-
-      // Encode Refs as Ints, to avoid table dependency issues. We'll convert back to Ref at the end.
-      encodeRuleReferences(transformRule);
-    } else if (transformRule.destCols.some(c => c.colId === null)) {
-      throw new Error('Column ids in transform rule must be filled when importing into an existing table');
-    }
-
-    await this._activeDoc.applyUserActions(docSession,
-      [['MakeImportTransformColumns', hiddenTableId, transformRule, false]]);
+    const transformDestTableId = intoNewTable ? null : destTableId;
+    const result = await this._activeDoc.applyUserActions(docSession, [[
+      'GenImporterView', hiddenTableId, transformDestTableId, transformRule,
+      {createViewSection: false, genAll: false, refsAsInts: true},
+    ]]);
+    transformRule = result.retValues[0].transformRule as TransformRule;

    if (!intoNewTable && mergeOptions && mergeOptions.mergeCols.length > 0) {
      await this._mergeAndFinishImport(docSession, hiddenTableId, destTableId, transformRule, mergeOptions);
@@ -430,6 +415,7 @@ export class ActiveDocImport {
    const hiddenTableData = fromTableDataAction(await this._activeDoc.fetchTable(docSession, hiddenTableId, true));
    const columnData: BulkColValues = {};

+    const srcCols = await this._activeDoc.getTableCols(docSession, hiddenTableId);
    const srcColIds = srcCols.map(c => c.id as string);

    // Only include destination columns that weren't skipped.
@@ -591,38 +577,6 @@ export class ActiveDocImport {
    return this._activeDoc.docStorage.decodeMarshalledDataFromTables(result);
  }

-  /**
-   * Returns a default TransformRule using column definitions from `destTableId`. If `destTableId`
-   * is null (in the case when the import destination is a new table), the `srcCols` are used instead.
-   *
-   * @param {TableRecordValue[]} srcCols Source column definitions.
-   * @param {string|null} destTableId The destination table id. If null, the destination is assumed
-   * to be a new table, and `srcCols` are used to build the transform rule.
-   * @returns {Promise<TransformRule>} The constructed transform rule.
-   */
-  private async _makeDefaultTransformRule(docSession: OptDocSession, srcCols: TableRecordValue[],
-                                          destTableId: string|null): Promise<TransformRule> {
-    const targetCols = destTableId ? await this._activeDoc.getTableCols(docSession, destTableId) : srcCols;
-    const destCols: TransformColumn[] = [];
-    const srcColIds = srcCols.map(c => c.id as string);
-
-    for (const {id, fields} of targetCols) {
-      destCols.push({
-        colId: destTableId ? id as string : null,
-        label: fields.label as string,
-        type: fields.type as string,
-        widgetOptions: fields.widgetOptions as string,
-        formula: srcColIds.includes(id as string) ? `$${id}` :  ''
-      });
-    }
-
-    return {
-      destTableId,
-      destCols,
-      sourceCols: srcColIds
-    };
-  }
-
  /**
   * This function removes temporary hidden tables which were created during the import process
   *
@@ -694,32 +648,6 @@ function isBlank(value: CellValue): boolean {
  return value === null || (typeof value === 'string' && value.trim().length === 0);
 }

-/**
- * Changes every Ref column to an Int column in `destCols`.
- *
- * Encoding references as ints can be useful when finishing imports to avoid
- * issues such as importing linked tables in the wrong order. When encoding references,
- * ActiveDocImport._fixReferences should be called at the end of importing to
- * decode Ints back to Refs.
- */
-function encodeRuleReferences({destCols}: TransformRule): void {
-  for (const col of destCols) {
-    const refTableId = gutil.removePrefix(col.type, "Ref:");
-    if (refTableId) {
-      col.type = 'Int';
-    }
-  }
-}
-
-// Helper function that strips import prefixes from columns in transform rules (if ids are present).
-function stripRulePrefixes({destCols}: TransformRule): void {
-  for (const col of destCols) {
-    const colId = col.colId;
-    if (colId && colId.startsWith(IMPORT_TRANSFORM_COLUMN_PREFIX)) {
-      col.colId = colId.slice(IMPORT_TRANSFORM_COLUMN_PREFIX.length);
-    }
-  }
-}

 // Helper function that returns new `colIds` with import prefixes stripped.
 function stripPrefixes(colIds: string[]): string[] {
--- a/app/server/lib/Export.ts
+++ b/app/server/lib/Export.ts
@@ -1,6 +1,7 @@
 import {ApiError} from 'app/common/ApiError';
 import {buildColFilter} from 'app/common/ColumnFilterFunc';
 import {TableDataAction} from 'app/common/DocActions';
+import {DocData} from 'app/common/DocData';
 import {DocumentSettings} from 'app/common/DocumentSettings';
 import * as gristTypes from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
@@ -11,6 +12,7 @@ import {schema, SchemaTypes} from 'app/common/schema';
 import {SortFunc} from 'app/common/SortFunc';
 import {Sort} from 'app/common/SortSpec';
 import {MetaRowRecord, MetaTableData} from 'app/common/TableData';
+import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFormatter';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
 import {docSessionFromRequest} from 'app/server/lib/DocSession';
@@ -23,23 +25,16 @@ import * as _ from 'underscore';
 type Access = (row: number) => any;

 // Helper interface with information about the column
-interface ExportColumn {
+export interface ExportColumn {
  id: number;
  colId: string;
  label: string;
  type: string;
-  widgetOptions: any;
+  formatter: BaseFormatter;
  parentPos: number;
+  description: string;
 }
-// helper for empty column
-const emptyCol: ExportColumn = {
-  id: 0,
-  colId: '',
-  label: '',
-  type: '',
-  widgetOptions: null,
-  parentPos: 0
-};
+
 /**
 * Bare data that is exported - used to convert to various formats.
 */
@@ -173,41 +168,33 @@ export async function exportTable(
  {metaTables}: {metaTables?: FilteredMetaTables} = {},
 ): Promise<ExportData> {
  metaTables = metaTables || await getMetaTables(activeDoc, req);
+  const docData = new DocData((tableId) => { throw new Error("Unexpected DocData fetch"); }, metaTables);
  const tables = safeTable(metaTables, '_grist_Tables');
+  const metaColumns = safeTable(metaTables, '_grist_Tables_column');
  checkTableAccess(tables, tableRef);
  const table = safeRecord(tables, tableRef);
-  const tableColumns = safeTable(metaTables, '_grist_Tables_column')
-    .getRecords()
+
+  // Select only columns that belong to this table.
+  const tableColumns = metaColumns.filterRecords({parentId: tableRef})
    // sort by parentPos and id, which should be the same order as in raw data
-    .sort((c1, c2) => nativeCompare(c1.parentPos, c2.parentPos) || nativeCompare(c1.id, c2.id))
-    // remove manual sort column
-    .filter(col => col.colId !== gristTypes.MANUALSORT);
+    .sort((c1, c2) => nativeCompare(c1.parentPos, c2.parentPos) || nativeCompare(c1.id, c2.id));
+
  // Produce a column description matching what user will see / expect to export
-  const tableColsById = _.indexBy(tableColumns, 'id');
-  const columns = tableColumns.map(tc => {
-    // remove all columns that don't belong to this table
-    if (tc.parentId !== tableRef) {
-      return emptyCol;
-    }
-    // remove all helpers
-    if (gristTypes.isHiddenCol(tc.colId)) {
-      return emptyCol;
-    }
+  const columns: ExportColumn[] = tableColumns
+  .filter(tc => !gristTypes.isHiddenCol(tc.colId))    // Exclude helpers
+  .map<ExportColumn>(tc => {
    // for reference columns, return display column, and copy settings from visible column
-    const displayCol = tableColsById[tc.displayCol || tc.id];
-    const colOptions = gutil.safeJsonParse(tc.widgetOptions, {});
-    const displayOptions = gutil.safeJsonParse(displayCol.widgetOptions, {});
-    const widgetOptions = Object.assign(displayOptions, colOptions);
+    const displayCol = metaColumns.getRecord(tc.displayCol) || tc;
    return {
      id: displayCol.id,
      colId: displayCol.colId,
      label: tc.label,
-      type: displayCol.type,
-      widgetOptions,
+      type: tc.type,
+      formatter: createFullFormatterFromDocData(docData, tc.id),
      parentPos: tc.parentPos,
-      description: displayCol.description,
+      description: tc.description,
    };
-  }).filter(tc => tc !== emptyCol);
+  });

  // fetch actual data
  const {tableData} = await activeDoc.fetchTable(docSessionFromRequest(req as RequestWithLogin), table.tableId, true);
@@ -251,37 +238,35 @@ export async function exportSection(
  {metaTables}: {metaTables?: FilteredMetaTables} = {},
 ): Promise<ExportData> {
  metaTables = metaTables || await getMetaTables(activeDoc, req);
+  const docData = new DocData((tableId) => { throw new Error("Unexpected DocData fetch"); }, metaTables);
  const viewSections = safeTable(metaTables, '_grist_Views_section');
  const viewSection = safeRecord(viewSections, viewSectionId);
  safe(viewSection.tableRef, `Cannot find or access table`);
  const tables = safeTable(metaTables, '_grist_Tables');
  checkTableAccess(tables, viewSection.tableRef);
  const table = safeRecord(tables, viewSection.tableRef);
-  const columns = safeTable(metaTables, '_grist_Tables_column')
-    .filterRecords({parentId: table.id});
+  const metaColumns = safeTable(metaTables, '_grist_Tables_column');
+  const columns = metaColumns.filterRecords({parentId: table.id});
  const viewSectionFields = safeTable(metaTables, '_grist_Views_section_field');
  const fields = viewSectionFields.filterRecords({parentId: viewSection.id});
  const savedFilters = safeTable(metaTables, '_grist_Filters')
    .filterRecords({viewSectionRef: viewSection.id});

-  const tableColsById = _.indexBy(columns, 'id');
  const fieldsByColRef = _.indexBy(fields, 'colRef');
  const savedFiltersByColRef = _.indexBy(savedFilters, 'colRef');
  const unsavedFiltersByColRef = _.indexBy(filters ?? [], 'colRef');

  // Produce a column description matching what user will see / expect to export
-  const viewify = (col: GristTablesColumn, field?: GristViewsSectionField) => {
-    const displayCol = tableColsById[field?.displayCol || col.displayCol || col.id];
-    const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {});
-    const fieldWidgetOptions = field ? gutil.safeJsonParse(field.widgetOptions, {}) : {};
+  const viewify = (col: GristTablesColumn, field?: GristViewsSectionField): ExportColumn => {
+    const displayCol = metaColumns.getRecord(field?.displayCol || col.displayCol) || col;
    return {
      id: displayCol.id,
      colId: displayCol.colId,
      label: col.label,
      type: col.type,
+      formatter: createFullFormatterFromDocData(docData, col.id, field?.id),
      parentPos: col.parentPos,
      description: col.description,
-      widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions),
    };
  };
  const buildFilters = (col: GristTablesColumn, field?: GristViewsSectionField) => {
@@ -297,14 +282,14 @@ export async function exportSection(
  const columnsForFilters = columns
    .filter(column => !gristTypes.isHiddenCol(column.colId))
    .map(column => buildFilters(column, fieldsByColRef[column.id]));
-  const viewColumns = _.sortBy(fields, 'parentPos')
-    .map((field) => viewify(tableColsById[field.colRef], field));
+  const viewColumns: ExportColumn[] = _.sortBy(fields, 'parentPos')
+    .map((field) => viewify(metaColumns.getRecord(field.colRef)!, field));

  // The columns named in sort order need to now become display columns
  sortSpec = sortSpec || gutil.safeJsonParse(viewSection.sortColRefs, []);
  sortSpec = sortSpec!.map((colSpec) => {
    const colRef = Sort.getColRef(colSpec);
-    const col = tableColsById[colRef];
+    const col = metaColumns.getRecord(colRef);
    if (!col) {
      return 0;
    }
--- a/app/server/lib/ExportCSV.ts
+++ b/app/server/lib/ExportCSV.ts
@@ -1,5 +1,4 @@
 import {ApiError} from 'app/common/ApiError';
-import {createFormatter} from 'app/common/ValueFormatter';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {DownloadOptions, ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export';
 import log from 'app/server/lib/log';
@@ -86,7 +85,7 @@ function convertToCsv({
 }: ExportData) {

  // create formatters for columns
-  const formatters = viewColumns.map(col => createFormatter(col.type, col.widgetOptions, docSettings));
+  const formatters = viewColumns.map(col => col.formatter);
  // Arrange the data into a row-indexed matrix, starting with column headers.
  const csvMatrix = [viewColumns.map(col => col.label)];
  // populate all the rows with values as strings
--- a/app/server/lib/ExportTableSchema.ts
+++ b/app/server/lib/ExportTableSchema.ts
@@ -1,18 +1,7 @@
 import * as express from 'express';
 import {ApiError} from 'app/common/ApiError';
-import {WidgetOptions} from 'app/common/WidgetOptions';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {DownloadOptions, exportTable} from 'app/server/lib/Export';
-
-interface ExportColumn {
-  id: number;
-  colId: string;
-  label: string;
-  type: string;
-  widgetOptions: WidgetOptions;
-  description?: string;
-  parentPos: number;
-}
+import {DownloadOptions, ExportColumn, exportTable} from 'app/server/lib/Export';

 interface FrictionlessFormat {
  name: string;
@@ -86,35 +75,36 @@ function columnsToTableSchema(

 function buildTypeField(col: ExportColumn, locale: string) {
  const type = col.type.split(':', 1)[0];
+  const widgetOptions = col.formatter.widgetOpts;
  switch (type) {
    case 'Text':
      return {
        type: 'string',
-        format: col.widgetOptions.widget === 'HyperLink' ? 'uri' : 'default',
+        format: widgetOptions.widget === 'HyperLink' ? 'uri' : 'default',
      };
    case 'Numeric':
      return {
        type: 'number',
-        bareNumber: col.widgetOptions?.numMode === 'decimal',
+        bareNumber: widgetOptions?.numMode === 'decimal',
        ...getNumberSeparators(locale),
      };
    case 'Integer':
      return {
        type: 'integer',
-        bareNumber: col.widgetOptions?.numMode === 'decimal',
+        bareNumber: widgetOptions?.numMode === 'decimal',
        groupChar: getNumberSeparators(locale).groupChar,
      };
    case 'Date':
      return {
        type: 'date',
        format: 'any',
-        gristFormat: col.widgetOptions?.dateFormat || 'YYYY-MM-DD',
+        gristFormat: widgetOptions?.dateFormat || 'YYYY-MM-DD',
      };
    case 'DateTime':
      return {
        type: 'datetime',
        format: 'any',
-        gristFormat: `${col.widgetOptions?.dateFormat} ${col.widgetOptions?.timeFormat}`,
+        gristFormat: `${widgetOptions?.dateFormat} ${widgetOptions?.timeFormat}`,
      };
    case 'Bool':
      return {
@@ -125,12 +115,12 @@ function buildTypeField(col: ExportColumn, locale: string) {
    case 'Choice':
      return {
        type: 'string',
-        constraints: {enum: col.widgetOptions?.choices},
+        constraints: {enum: widgetOptions?.choices},
      };
    case 'ChoiceList':
      return {
        type: 'array',
-        constraints: {enum: col.widgetOptions?.choices},
+        constraints: {enum: widgetOptions?.choices},
      };
    case 'Reference':
      return {type: 'string'};
@@ -148,4 +138,4 @@ function getNumberSeparators(locale: string) {
    groupChar: parts.find(obj => obj.type === 'group')?.value,
    decimalChar: parts.find(obj => obj.type === 'decimal')?.value,
  };
-}
+}
--- a/app/server/lib/ExportXLSX.ts
+++ b/app/server/lib/ExportXLSX.ts
@@ -130,7 +130,7 @@ async function convertToExcel(tables: ExportData[], testDates: boolean) {
    const { columns, rowIds, access, tableName } = table;
    const ws = wb.addWorksheet(sanitizeWorksheetName(tableName));
    // Build excel formatters.
-    const formatters = columns.map(col => createExcelFormatter(col.type, col.widgetOptions));
+    const formatters = columns.map(col => createExcelFormatter(col.formatter.type, col.formatter.widgetOpts));
    // Generate headers for all columns with correct styles for whole column.
    // Actual header style for a first row will be overwritten later.
    ws.columns = columns.map((col, c) => ({ header: col.label, style: formatters[c].style() }));