(core) Fix imports into reference columns, and support two ways to import Numeric as a reference.

Summary:
- When importing into a Ref column, use lookupOne() formula for correct previews.
- When selecting columns to import into a Ref column, now a Numeric column like
  'Order' will produce two options: "Order" and "Order (as row ID)".
- Fixes exports to correct the formatting of visible columns. This addresses multiple bugs:
  1. Formatting wasn't used, e.g. a Ref showing a custom-formatted date was still presented as YYYY-MM-DD in CSVs.
  2. Ref showing a Numeric column was formatted as if a row ID (e.g. `Table1[1.5]`), which is very wrong.
- If importing into a table that doesn't have a primary view, don't switch page after import.

Refactorings:
- Generalize GenImporterView to be usable in more cases; removed near-duplicated logic from node side
- Some other refactoring in importing code.
- Fix field/column option selection in ValueParser
- Add NUM() helper to turn integer-valued floats into ints, useful for "as row ID" lookups.

Test Plan: Added test cases for imports into reference columns, updated Exports test fixtures.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3875

app/client/components/Importer.ts

@@ -19,7 +19,8 @@ import {openFilePicker} from 'app/client/ui/FileDialog';
 import {bigBasicButton, bigPrimaryButton} from 'app/client/ui2018/buttons';
 import {testId, theme, vars} from 'app/client/ui2018/cssVars';
 import {icon} from 'app/client/ui2018/icons';
-import {IOptionFull, linkSelect, menu, menuDivider, menuItem, multiSelect} from 'app/client/ui2018/menus';
+import {IOptionFull, linkSelect, menu,
+        menuDivider, menuItem, multiSelect} from 'app/client/ui2018/menus';
 import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals';
 import {loadingSpinner} from 'app/client/ui2018/loaders';
 import {openFormulaEditor} from 'app/client/widgets/FormulaEditor';
@@ -188,30 +189,29 @@ export class Importer extends DisposableWithEvents {
     ...use(this._gristDoc.docModel.visibleTableIds.getObservable()).map((id) => ({value: id, label: id})),
   ]);
 
-  // Source column labels for the selected import source, keyed by column id.
+  // List of transform fields, i.e. those formula fields of the transform section whose values
-  private _sourceColLabelsById = Computed.create(this, this._sourceInfoSelected, (use, info) => {
+  // will be used to populate the destination columns.
-    if (!info || use(info.sourceSection._isDeleted)) { return null; }
+  private _transformFields: Computed<ViewFieldRec[]|null> = Computed.create(
-
+      this, this._sourceInfoSelected, (use, info) => {
-    const fields = use(use(info.sourceSection.viewFields).getObservable());
+    const section = info && use(info.transformSection);
-    return new Map(fields.map(f => [use(use(f.column).colId), use(use(f.column).label)]));
+    if (!section || use(section._isDeleted)) { return null; }
+    return use(use(section.viewFields).getObservable());
   });
 
-  // Transform section columns of the selected source.
+  // Prepare a Map, mapping of colRef of each transform column to the set of options to offer in
-  private _transformSectionCols = Computed.create(this, this._sourceInfoSelected, (use, info) => {
+  // the dropdown. The options are represented as a Map too, mapping formula to label.
-    if (!info) { return null; }
+  private _transformColImportOptions: Computed<Map<number, Map<string, string>>> = Computed.create(
-
+      this, this._transformFields, this._sourceInfoSelected, (use, fields, info) => {
-    const transformSection = use(info.transformSection);
+    if (!fields || !info) { return new Map(); }
-    if (!transformSection || use(transformSection._isDeleted)) { return null; }
+    return new Map(fields.map(f =>
-
+      [use(f.colRef), this._makeImportOptionsForCol(use(f.column), info)]));
-    const fields = use(use(transformSection.viewFields).getObservable());
-    return fields.map(f => use(f.column));
   });
 
-  // List of destination fields that aren't mapped to a source column.
+  // List of labels of destination columns that aren't mapped to a source column, i.e. transform
-  private _unmatchedFields = Computed.create(this, this._transformSectionCols, (use, cols) => {
+  // columns with empty formulas.
-    if (!cols) { return null; }
+  private _unmatchedFields: Computed<string[]|undefined> = Computed.create(
-
+      this, this._transformFields, (use, fields) => {
-    return cols.filter(c => use(c.formula).trim() === '').map(c => c.label());
+    return fields?.filter(f => (use(use(f.column).formula).trim() === '')).map(f => use(f.label));
   });
 
   // null tells to use the built-in file picker.
@@ -299,9 +299,9 @@ export class Importer extends DisposableWithEvents {
     sourceInfo.transformSection.set(null);
 
     const genImporterViewPromise = this._gristDoc.docData.sendAction(
-      ['GenImporterView', sourceInfo.hiddenTableId, sourceInfo.destTableId.get(), null]);
+      ['GenImporterView', sourceInfo.hiddenTableId, sourceInfo.destTableId.get(), null, null]);
     sourceInfo.lastGenImporterViewPromise = genImporterViewPromise;
-    const transformSectionRef = await genImporterViewPromise;
+    const transformSectionRef = (await genImporterViewPromise).viewSectionRef;
 
     // If the request is superseded by a newer request, or the Importer is disposed, do nothing.
     if (this.isDisposed() || sourceInfo.lastGenImporterViewPromise !== genImporterViewPromise) {
@@ -444,7 +444,11 @@ export class Importer extends DisposableWithEvents {
 
     if (importResult.tables[0]?.hiddenTableId) {
       const tableRowModel = this._gristDoc.docModel.dataTables[importResult.tables[0].hiddenTableId].tableMetaRow;
-      await this._gristDoc.openDocPage(tableRowModel.primaryViewId());
+      const primaryViewId = tableRowModel.primaryViewId();
+      if (primaryViewId) {
+        // Switch page if there is a sensible one to switch to.
+        await this._gristDoc.openDocPage(primaryViewId);
+      }
     }
     this._screen.close();
     this.dispose();
@@ -685,49 +689,14 @@ export class Importer extends DisposableWithEvents {
                           ),
                           cssDestinationFieldSettings(
                             icon('Dots'),
-                            menu(
+                            menu(() => this._makeImportOptionsMenu(field.origCol.peek(), info),
-                              () => {
-                                const sourceColId = field.origCol().id();
-                                const sourceColIdsAndLabels = [...this._sourceColLabelsById.get()!.entries()];
-                                return [
-                                  menuItem(
-                                    async () => {
-                                      await this._gristDoc.clearColumns([sourceColId], {keepType: true});
-                                      await this._updateImportDiff(info);
-                                    },
-                                    'Skip',
-                                    testId('importer-column-match-menu-item')
-                                  ),
-                                  menuDivider(),
-                                  ...sourceColIdsAndLabels.map(([id, label]) =>
-                                    menuItem(
-                                      async () => {
-                                        await this._setColumnFormula(sourceColId, '$' + id);
-                                        await this._updateImportDiff(info);
-                                      },
-                                      label,
-                                      testId('importer-column-match-menu-item')
-                                    ),
-                                  ),
-                                  testId('importer-column-match-menu'),
-                                ];
-                              },
                               { placement: 'right-start' },
                             ),
                             testId('importer-column-match-destination-settings')
                           ),
                           testId('importer-column-match-destination')
                         ),
-                        dom.domComputed(use => dom.create(
+                        dom.create(owner => this._buildColMappingFormula(owner, field, info)),
-                          this._buildColMappingFormula.bind(this),
-                          use(field.column),
-                          (elem: Element) => this._activateFormulaEditor(
-                            elem,
-                            field,
-                            () => this._updateImportDiff(info),
-                          ),
-                          'Skip'
-                        )),
                         testId('importer-column-match-source-destination'),
                       )
                     )),
@@ -776,6 +745,52 @@ export class Importer extends DisposableWithEvents {
     this._screen.render(content, {fullscreen: true});
   }
 
+  private _makeImportOptionsForCol(transformCol: ColumnRec, info: SourceInfo) {
+    const options = new Map<string, string>();  // Maps formula to label.
+    const importedFields = info.sourceSection.viewFields.peek().peek();
+
+    // Reference columns are populated using lookup formulas, so figure out now if this is a
+    // reference column, and if so, its destination table and the lookup column ID.
+    const refTable = transformCol.refTable.peek();
+    const refTableId = refTable ? refTable.tableId.peek() : undefined;
+    const visibleColId = transformCol.visibleColModel.peek().colId.peek();
+    const isRefDest = Boolean(info.destTableId.get() && transformCol.pureType.peek() === 'Ref');
+
+    for (const f of importedFields) {
+      const importedCol = f.column.peek();
+      const colId = importedCol.colId.peek();
+      const colLabel = importedCol.label.peek();
+      if (isRefDest && visibleColId) {
+        const formula = `${refTableId}.lookupOne(${visibleColId}=$${colId}) or ($${colId} and str($${colId}))`;
+        options.set(formula, colLabel);
+      } else {
+        options.set(`$${colId}`, colLabel);
+      }
+      if (isRefDest && ['Numeric', 'Int'].includes(importedCol.type.peek())) {
+        options.set(`${refTableId}.lookupOne(id=NUM($${colId})) or ($${colId} and str(NUM($${colId})))`,
+          `${colLabel} (as row ID)`);
+      }
+    }
+    return options;
+  }
+
+  private _makeImportOptionsMenu(transformCol: ColumnRec, info: SourceInfo) {
+    const transformColRef = transformCol.id();
+    const options = this._transformColImportOptions.get().get(transformCol.getRowId());
+    return [
+      menuItem(() => this._setColumnFormula(transformColRef, null, info),
+        'Skip',
+        testId('importer-column-match-menu-item')),
+      menuDivider(),
+      ...Array.from(options || [], ([formula, label]) =>
+        menuItem(() => this._setColumnFormula(transformColRef, formula, info),
+          label,
+          testId('importer-column-match-menu-item'))
+      ),
+      testId('importer-column-match-menu'),
+    ];
+  }
+
   private _addFocusLayer(container: HTMLElement) {
     dom.autoDisposeElem(container, new FocusLayer({
       defaultFocusElem: container,
@@ -787,10 +802,14 @@ export class Importer extends DisposableWithEvents {
   /**
    * Updates the formula on column `colRef` to `formula`.
    */
-  private async _setColumnFormula(colRef: number, formula: string): Promise<void> {
+  private async _setColumnFormula(transformColRef: number, formula: string|null, info: SourceInfo) {
-    return this._gristDoc.docModel.columns.sendTableAction(
+    if (formula === null) {
-      ['UpdateRecord', colRef, { formula, isFormula: true }]
+      await this._gristDoc.clearColumns([transformColRef], {keepType: true});
-    );
+    } else {
+      await this._gristDoc.docModel.columns.sendTableAction(
+        ['UpdateRecord', transformColRef, { formula, isFormula: true }]);
+    }
+    await this._updateImportDiff(info);
   }
 
   /**
@@ -841,26 +860,20 @@ export class Importer extends DisposableWithEvents {
    * in the column mapping section of Importer. On click, opens
    * an editor for the formula for `column`.
    */
-  private _buildColMappingFormula(_owner: MultiHolder, column: ColumnRec, buildEditor: (e: Element) => void,
+  private _buildColMappingFormula(owner: MultiHolder, field: ViewFieldRec, info: SourceInfo) {
-                                  placeholder: string) {
+    const displayFormula = Computed.create(owner, use => {
-    const formatFormula = (formula: string) => {
+      const column = use(field.column);
-      const sourceColLabels = this._sourceColLabelsById.get();
+      const formula = use(column.formula);
-      if (!sourceColLabels) { return formula; }
+      const importOptions = use(this._transformColImportOptions).get(column.getRowId());
+      return importOptions?.get(formula) ?? formula;
+    });
 
-      formula = formula.trim();
+    return cssFieldFormula(displayFormula,
-      if (formula.startsWith('$') && sourceColLabels.has(formula.slice(1))) {
+      {gristTheme: this._gristDoc.currentTheme, placeholder: 'Skip', maxLines: 1},
-        // For simple formulas that only reference a source column id, show the source column label.
-        return sourceColLabels.get(formula.slice(1))!;
-      }
-
-      return formula;
-    };
-
-    return cssFieldFormula(use => formatFormula(use(column.formula)),
-      {gristTheme: this._gristDoc.currentTheme, placeholder, maxLines: 1},
       dom.cls('disabled'),
       {tabIndex: '-1'},
-      dom.on('focus', (_ev, elem) => buildEditor(elem)),
+      dom.on('focus', (_ev, elem) =>
+        this._activateFormulaEditor(elem, field, () => this._updateImportDiff(info))),
       testId('importer-column-match-formula'),
     );
   }

app/common/ValueFormatter.ts

@@ -70,7 +70,7 @@ export type IsRightTypeFunc = (value: CellValue) => boolean;
 export class BaseFormatter {
   protected isRightType: IsRightTypeFunc;
 
-  constructor(public type: string, public widgetOpts: object, public docSettings: DocumentSettings) {
+  constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) {
     this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) ||
       gristTypes.isRightType('Any')!;
   }
@@ -144,7 +144,7 @@ export interface DateFormatOptions {
 }
 
 class DateFormatter extends BaseFormatter {
-  private _dateTimeFormat: string;
+  protected _dateTimeFormat: string;
   private _timezone: string;
 
   constructor(type: string, widgetOpts: DateFormatOptions, docSettings: DocumentSettings, timezone: string = 'UTC') {
@@ -194,9 +194,11 @@ export interface DateTimeFormatOptions extends DateFormatOptions {
 class DateTimeFormatter extends DateFormatter {
   constructor(type: string, widgetOpts: DateTimeFormatOptions, docSettings: DocumentSettings) {
     const timezone = gutil.removePrefix(type, "DateTime:") || '';
+    // Pass up the original widgetOpts. It's helpful to have them available; e.g. ExcelFormatter
+    // takes options from an initialized ValueFormatter.
+    super(type, widgetOpts, docSettings, timezone);
     const timeFormat = widgetOpts.timeFormat === undefined ? 'h:mma' : widgetOpts.timeFormat;
-    const dateFormat = (widgetOpts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
+    this._dateTimeFormat = (widgetOpts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
-    super(type, {dateFormat}, docSettings, timezone);
   }
 }
 

app/common/ValueParser.ts

@@ -268,7 +268,8 @@ export function createParserOrFormatterArguments(
   const col = columnsTable.getRecord(colRef)!;
   let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col;
   if (fieldRef) {
-    fieldOrCol = fieldsTable.getRecord(fieldRef) || col;
+    const field = fieldsTable.getRecord(fieldRef);
+    fieldOrCol = field?.widgetOptions ? field : col;
   }
 
   return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol);

app/server/lib/ActiveDocImport.ts

@@ -5,11 +5,11 @@ import * as _ from 'underscore';
 
 import {ColumnDelta, createEmptyActionSummary} from 'app/common/ActionSummary';
 import {ApplyUAResult, DataSourceTransformed, ImportOptions, ImportResult, ImportTableResult,
-        MergeOptions, MergeOptionsMap, MergeStrategy, SKIP_TABLE, TransformColumn,
+        MergeOptions, MergeOptionsMap, MergeStrategy, SKIP_TABLE,
         TransformRule,
         TransformRuleMap} from 'app/common/ActiveDocAPI';
 import {ApiError} from 'app/common/ApiError';
-import {BulkColValues, CellValue, fromTableDataAction, TableRecordValue, UserAction} from 'app/common/DocActions';
+import {BulkColValues, CellValue, fromTableDataAction, UserAction} from 'app/common/DocActions';
 import * as gutil from 'app/common/gutil';
 import {DocStateComparison} from 'app/common/UserAPI';
 import {guessColInfoForImports} from 'app/common/ValueGuesser';
@@ -339,9 +339,9 @@ export class ActiveDocImport {
       if (isHidden) {
         // Generate formula columns, view sections, etc
         const results: ApplyUAResult = await this._activeDoc.applyUserActions(docSession,
-          [['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null]]);
+          [['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null, null]]);
 
-        transformSectionRef = results.retValues[0];
+        transformSectionRef = results.retValues[0].viewSectionRef;
         createdTableId = hiddenTableId;
 
       } else {
@@ -391,36 +391,21 @@ export class ActiveDocImport {
    * the source and destination table.
    * @returns {string} The table id of the new or updated destination table.
    */
-  private async _transformAndFinishImport(docSession: OptDocSession,
+  private async _transformAndFinishImport(
-                                          hiddenTableId: string, destTableId: string,
+    docSession: OptDocSession,
-                                          intoNewTable: boolean, transformRule: TransformRule|null,
+    hiddenTableId: string, destTableId: string,
-                                          mergeOptions: MergeOptions|null): Promise<string> {
+    intoNewTable: boolean, transformRule: TransformRule|null,
+    mergeOptions: MergeOptions|null
+  ): Promise<string> {
     log.info("ActiveDocImport._transformAndFinishImport(%s, %s, %s, %s, %s)",
       hiddenTableId, destTableId, intoNewTable, transformRule, mergeOptions);
-    const srcCols = await this._activeDoc.getTableCols(docSession, hiddenTableId);
 
-    // Use a default transform rule if one was not provided by the client.
+    const transformDestTableId = intoNewTable ? null : destTableId;
-    if (!transformRule) {
+    const result = await this._activeDoc.applyUserActions(docSession, [[
-      const transformDest = intoNewTable ? null : destTableId;
+      'GenImporterView', hiddenTableId, transformDestTableId, transformRule,
-      transformRule = await this._makeDefaultTransformRule(docSession, srcCols, transformDest);
+      {createViewSection: false, genAll: false, refsAsInts: true},
-    }
+    ]]);
-
+    transformRule = result.retValues[0].transformRule as TransformRule;
-    // Transform rules from client may have prefixed column ids, so we need to strip them.
-    stripRulePrefixes(transformRule);
-
-    if (intoNewTable) {
-      // Transform rules for new tables don't have filled in destination column ids.
-      const result = await this._activeDoc.applyUserActions(docSession, [['FillTransformRuleColIds', transformRule]]);
-      transformRule = result.retValues[0] as TransformRule;
-
-      // Encode Refs as Ints, to avoid table dependency issues. We'll convert back to Ref at the end.
-      encodeRuleReferences(transformRule);
-    } else if (transformRule.destCols.some(c => c.colId === null)) {
-      throw new Error('Column ids in transform rule must be filled when importing into an existing table');
-    }
-
-    await this._activeDoc.applyUserActions(docSession,
-      [['MakeImportTransformColumns', hiddenTableId, transformRule, false]]);
 
     if (!intoNewTable && mergeOptions && mergeOptions.mergeCols.length > 0) {
       await this._mergeAndFinishImport(docSession, hiddenTableId, destTableId, transformRule, mergeOptions);
@@ -430,6 +415,7 @@ export class ActiveDocImport {
     const hiddenTableData = fromTableDataAction(await this._activeDoc.fetchTable(docSession, hiddenTableId, true));
     const columnData: BulkColValues = {};
 
+    const srcCols = await this._activeDoc.getTableCols(docSession, hiddenTableId);
     const srcColIds = srcCols.map(c => c.id as string);
 
     // Only include destination columns that weren't skipped.
@@ -591,38 +577,6 @@ export class ActiveDocImport {
     return this._activeDoc.docStorage.decodeMarshalledDataFromTables(result);
   }
 
-  /**
-   * Returns a default TransformRule using column definitions from `destTableId`. If `destTableId`
-   * is null (in the case when the import destination is a new table), the `srcCols` are used instead.
-   *
-   * @param {TableRecordValue[]} srcCols Source column definitions.
-   * @param {string|null} destTableId The destination table id. If null, the destination is assumed
-   * to be a new table, and `srcCols` are used to build the transform rule.
-   * @returns {Promise<TransformRule>} The constructed transform rule.
-   */
-  private async _makeDefaultTransformRule(docSession: OptDocSession, srcCols: TableRecordValue[],
-                                          destTableId: string|null): Promise<TransformRule> {
-    const targetCols = destTableId ? await this._activeDoc.getTableCols(docSession, destTableId) : srcCols;
-    const destCols: TransformColumn[] = [];
-    const srcColIds = srcCols.map(c => c.id as string);
-
-    for (const {id, fields} of targetCols) {
-      destCols.push({
-        colId: destTableId ? id as string : null,
-        label: fields.label as string,
-        type: fields.type as string,
-        widgetOptions: fields.widgetOptions as string,
-        formula: srcColIds.includes(id as string) ? `$${id}` :  ''
-      });
-    }
-
-    return {
-      destTableId,
-      destCols,
-      sourceCols: srcColIds
-    };
-  }
-
   /**
    * This function removes temporary hidden tables which were created during the import process
    *
@@ -694,32 +648,6 @@ function isBlank(value: CellValue): boolean {
   return value === null || (typeof value === 'string' && value.trim().length === 0);
 }
 
-/**
- * Changes every Ref column to an Int column in `destCols`.
- *
- * Encoding references as ints can be useful when finishing imports to avoid
- * issues such as importing linked tables in the wrong order. When encoding references,
- * ActiveDocImport._fixReferences should be called at the end of importing to
- * decode Ints back to Refs.
- */
-function encodeRuleReferences({destCols}: TransformRule): void {
-  for (const col of destCols) {
-    const refTableId = gutil.removePrefix(col.type, "Ref:");
-    if (refTableId) {
-      col.type = 'Int';
-    }
-  }
-}
-
-// Helper function that strips import prefixes from columns in transform rules (if ids are present).
-function stripRulePrefixes({destCols}: TransformRule): void {
-  for (const col of destCols) {
-    const colId = col.colId;
-    if (colId && colId.startsWith(IMPORT_TRANSFORM_COLUMN_PREFIX)) {
-      col.colId = colId.slice(IMPORT_TRANSFORM_COLUMN_PREFIX.length);
-    }
-  }
-}
 
 // Helper function that returns new `colIds` with import prefixes stripped.
 function stripPrefixes(colIds: string[]): string[] {

app/server/lib/Export.ts

@@ -1,6 +1,7 @@
 import {ApiError} from 'app/common/ApiError';
 import {buildColFilter} from 'app/common/ColumnFilterFunc';
 import {TableDataAction} from 'app/common/DocActions';
+import {DocData} from 'app/common/DocData';
 import {DocumentSettings} from 'app/common/DocumentSettings';
 import * as gristTypes from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
@@ -11,6 +12,7 @@ import {schema, SchemaTypes} from 'app/common/schema';
 import {SortFunc} from 'app/common/SortFunc';
 import {Sort} from 'app/common/SortSpec';
 import {MetaRowRecord, MetaTableData} from 'app/common/TableData';
+import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFormatter';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
 import {docSessionFromRequest} from 'app/server/lib/DocSession';
@@ -23,23 +25,16 @@ import * as _ from 'underscore';
 type Access = (row: number) => any;
 
 // Helper interface with information about the column
-interface ExportColumn {
+export interface ExportColumn {
   id: number;
   colId: string;
   label: string;
   type: string;
-  widgetOptions: any;
+  formatter: BaseFormatter;
   parentPos: number;
+  description: string;
 }
-// helper for empty column
+
-const emptyCol: ExportColumn = {
-  id: 0,
-  colId: '',
-  label: '',
-  type: '',
-  widgetOptions: null,
-  parentPos: 0
-};
 /**
  * Bare data that is exported - used to convert to various formats.
  */
@@ -173,41 +168,33 @@ export async function exportTable(
   {metaTables}: {metaTables?: FilteredMetaTables} = {},
 ): Promise<ExportData> {
   metaTables = metaTables || await getMetaTables(activeDoc, req);
+  const docData = new DocData((tableId) => { throw new Error("Unexpected DocData fetch"); }, metaTables);
   const tables = safeTable(metaTables, '_grist_Tables');
+  const metaColumns = safeTable(metaTables, '_grist_Tables_column');
   checkTableAccess(tables, tableRef);
   const table = safeRecord(tables, tableRef);
-  const tableColumns = safeTable(metaTables, '_grist_Tables_column')
+
-    .getRecords()
+  // Select only columns that belong to this table.
+  const tableColumns = metaColumns.filterRecords({parentId: tableRef})
     // sort by parentPos and id, which should be the same order as in raw data
-    .sort((c1, c2) => nativeCompare(c1.parentPos, c2.parentPos) || nativeCompare(c1.id, c2.id))
+    .sort((c1, c2) => nativeCompare(c1.parentPos, c2.parentPos) || nativeCompare(c1.id, c2.id));
-    // remove manual sort column
+
-    .filter(col => col.colId !== gristTypes.MANUALSORT);
   // Produce a column description matching what user will see / expect to export
-  const tableColsById = _.indexBy(tableColumns, 'id');
+  const columns: ExportColumn[] = tableColumns
-  const columns = tableColumns.map(tc => {
+  .filter(tc => !gristTypes.isHiddenCol(tc.colId))    // Exclude helpers
-    // remove all columns that don't belong to this table
+  .map<ExportColumn>(tc => {
-    if (tc.parentId !== tableRef) {
-      return emptyCol;
-    }
-    // remove all helpers
-    if (gristTypes.isHiddenCol(tc.colId)) {
-      return emptyCol;
-    }
     // for reference columns, return display column, and copy settings from visible column
-    const displayCol = tableColsById[tc.displayCol || tc.id];
+    const displayCol = metaColumns.getRecord(tc.displayCol) || tc;
-    const colOptions = gutil.safeJsonParse(tc.widgetOptions, {});
-    const displayOptions = gutil.safeJsonParse(displayCol.widgetOptions, {});
-    const widgetOptions = Object.assign(displayOptions, colOptions);
     return {
       id: displayCol.id,
       colId: displayCol.colId,
       label: tc.label,
-      type: displayCol.type,
+      type: tc.type,
-      widgetOptions,
+      formatter: createFullFormatterFromDocData(docData, tc.id),
       parentPos: tc.parentPos,
-      description: displayCol.description,
+      description: tc.description,
     };
-  }).filter(tc => tc !== emptyCol);
+  });
 
   // fetch actual data
   const {tableData} = await activeDoc.fetchTable(docSessionFromRequest(req as RequestWithLogin), table.tableId, true);
@@ -251,37 +238,35 @@ export async function exportSection(
   {metaTables}: {metaTables?: FilteredMetaTables} = {},
 ): Promise<ExportData> {
   metaTables = metaTables || await getMetaTables(activeDoc, req);
+  const docData = new DocData((tableId) => { throw new Error("Unexpected DocData fetch"); }, metaTables);
   const viewSections = safeTable(metaTables, '_grist_Views_section');
   const viewSection = safeRecord(viewSections, viewSectionId);
   safe(viewSection.tableRef, `Cannot find or access table`);
   const tables = safeTable(metaTables, '_grist_Tables');
   checkTableAccess(tables, viewSection.tableRef);
   const table = safeRecord(tables, viewSection.tableRef);
-  const columns = safeTable(metaTables, '_grist_Tables_column')
+  const metaColumns = safeTable(metaTables, '_grist_Tables_column');
-    .filterRecords({parentId: table.id});
+  const columns = metaColumns.filterRecords({parentId: table.id});
   const viewSectionFields = safeTable(metaTables, '_grist_Views_section_field');
   const fields = viewSectionFields.filterRecords({parentId: viewSection.id});
   const savedFilters = safeTable(metaTables, '_grist_Filters')
     .filterRecords({viewSectionRef: viewSection.id});
 
-  const tableColsById = _.indexBy(columns, 'id');
   const fieldsByColRef = _.indexBy(fields, 'colRef');
   const savedFiltersByColRef = _.indexBy(savedFilters, 'colRef');
   const unsavedFiltersByColRef = _.indexBy(filters ?? [], 'colRef');
 
   // Produce a column description matching what user will see / expect to export
-  const viewify = (col: GristTablesColumn, field?: GristViewsSectionField) => {
+  const viewify = (col: GristTablesColumn, field?: GristViewsSectionField): ExportColumn => {
-    const displayCol = tableColsById[field?.displayCol || col.displayCol || col.id];
+    const displayCol = metaColumns.getRecord(field?.displayCol || col.displayCol) || col;
-    const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {});
-    const fieldWidgetOptions = field ? gutil.safeJsonParse(field.widgetOptions, {}) : {};
     return {
       id: displayCol.id,
       colId: displayCol.colId,
       label: col.label,
       type: col.type,
+      formatter: createFullFormatterFromDocData(docData, col.id, field?.id),
       parentPos: col.parentPos,
       description: col.description,
-      widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions),
     };
   };
   const buildFilters = (col: GristTablesColumn, field?: GristViewsSectionField) => {
@@ -297,14 +282,14 @@ export async function exportSection(
   const columnsForFilters = columns
     .filter(column => !gristTypes.isHiddenCol(column.colId))
     .map(column => buildFilters(column, fieldsByColRef[column.id]));
-  const viewColumns = _.sortBy(fields, 'parentPos')
+  const viewColumns: ExportColumn[] = _.sortBy(fields, 'parentPos')
-    .map((field) => viewify(tableColsById[field.colRef], field));
+    .map((field) => viewify(metaColumns.getRecord(field.colRef)!, field));
 
   // The columns named in sort order need to now become display columns
   sortSpec = sortSpec || gutil.safeJsonParse(viewSection.sortColRefs, []);
   sortSpec = sortSpec!.map((colSpec) => {
     const colRef = Sort.getColRef(colSpec);
-    const col = tableColsById[colRef];
+    const col = metaColumns.getRecord(colRef);
     if (!col) {
       return 0;
     }

app/server/lib/ExportCSV.ts

@@ -1,5 +1,4 @@
 import {ApiError} from 'app/common/ApiError';
-import {createFormatter} from 'app/common/ValueFormatter';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {DownloadOptions, ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export';
 import log from 'app/server/lib/log';
@@ -86,7 +85,7 @@ function convertToCsv({
 }: ExportData) {
 
   // create formatters for columns
-  const formatters = viewColumns.map(col => createFormatter(col.type, col.widgetOptions, docSettings));
+  const formatters = viewColumns.map(col => col.formatter);
   // Arrange the data into a row-indexed matrix, starting with column headers.
   const csvMatrix = [viewColumns.map(col => col.label)];
   // populate all the rows with values as strings

app/server/lib/ExportTableSchema.ts

@@ -1,18 +1,7 @@
 import * as express from 'express';
 import {ApiError} from 'app/common/ApiError';
-import {WidgetOptions} from 'app/common/WidgetOptions';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {DownloadOptions, exportTable} from 'app/server/lib/Export';
+import {DownloadOptions, ExportColumn, exportTable} from 'app/server/lib/Export';
-
-interface ExportColumn {
-  id: number;
-  colId: string;
-  label: string;
-  type: string;
-  widgetOptions: WidgetOptions;
-  description?: string;
-  parentPos: number;
-}
 
 interface FrictionlessFormat {
   name: string;
@@ -86,35 +75,36 @@ function columnsToTableSchema(
 
 function buildTypeField(col: ExportColumn, locale: string) {
   const type = col.type.split(':', 1)[0];
+  const widgetOptions = col.formatter.widgetOpts;
   switch (type) {
     case 'Text':
       return {
         type: 'string',
-        format: col.widgetOptions.widget === 'HyperLink' ? 'uri' : 'default',
+        format: widgetOptions.widget === 'HyperLink' ? 'uri' : 'default',
       };
     case 'Numeric':
       return {
         type: 'number',
-        bareNumber: col.widgetOptions?.numMode === 'decimal',
+        bareNumber: widgetOptions?.numMode === 'decimal',
         ...getNumberSeparators(locale),
       };
     case 'Integer':
       return {
         type: 'integer',
-        bareNumber: col.widgetOptions?.numMode === 'decimal',
+        bareNumber: widgetOptions?.numMode === 'decimal',
         groupChar: getNumberSeparators(locale).groupChar,
       };
     case 'Date':
       return {
         type: 'date',
         format: 'any',
-        gristFormat: col.widgetOptions?.dateFormat || 'YYYY-MM-DD',
+        gristFormat: widgetOptions?.dateFormat || 'YYYY-MM-DD',
       };
     case 'DateTime':
       return {
         type: 'datetime',
         format: 'any',
-        gristFormat: `${col.widgetOptions?.dateFormat} ${col.widgetOptions?.timeFormat}`,
+        gristFormat: `${widgetOptions?.dateFormat} ${widgetOptions?.timeFormat}`,
       };
     case 'Bool':
       return {
@@ -125,12 +115,12 @@ function buildTypeField(col: ExportColumn, locale: string) {
     case 'Choice':
       return {
         type: 'string',
-        constraints: {enum: col.widgetOptions?.choices},
+        constraints: {enum: widgetOptions?.choices},
       };
     case 'ChoiceList':
       return {
         type: 'array',
-        constraints: {enum: col.widgetOptions?.choices},
+        constraints: {enum: widgetOptions?.choices},
       };
     case 'Reference':
       return {type: 'string'};
@@ -148,4 +138,4 @@ function getNumberSeparators(locale: string) {
     groupChar: parts.find(obj => obj.type === 'group')?.value,
     decimalChar: parts.find(obj => obj.type === 'decimal')?.value,
   };
-}
+}

app/server/lib/ExportXLSX.ts

@@ -130,7 +130,7 @@ async function convertToExcel(tables: ExportData[], testDates: boolean) {
     const { columns, rowIds, access, tableName } = table;
     const ws = wb.addWorksheet(sanitizeWorksheetName(tableName));
     // Build excel formatters.
-    const formatters = columns.map(col => createExcelFormatter(col.type, col.widgetOptions));
+    const formatters = columns.map(col => createExcelFormatter(col.formatter.type, col.formatter.widgetOpts));
     // Generate headers for all columns with correct styles for whole column.
     // Actual header style for a first row will be overwritten later.
     ws.columns = columns.map((col, c) => ({ header: col.label, style: formatters[c].style() }));

sandbox/grist/functions/math.py

@@ -5,10 +5,9 @@ from __future__ import absolute_import
 import datetime
 import math as _math
 import operator
-import os
 import random
 import uuid
-from functools import reduce
+from functools import reduce  # pylint: disable=redefined-builtin
 
 from six.moves import zip, xrange
 import six
@@ -491,6 +490,25 @@ def MULTINOMIAL(value1, *more_values):
     res *= COMBIN(s, v)
   return res
 
+def NUM(value):
+  """
+  For a Python floating-point value that's actually an integer, returns a Python integer type.
+  Otherwise, returns the value unchanged. This is helpful sometimes when a value comes from a
+  Numeric Grist column (represented as floats), but when int values are actually expected.
+
+  >>> NUM(-17.0)
+  -17
+  >>> NUM(1.5)
+  1.5
+  >>> NUM(4)
+  4
+  >>> NUM("NA")
+  'NA'
+  """
+  if isinstance(value, float) and value.is_integer():
+    return int(value)
+  return value
+
 def ODD(value):
   """
   Rounds a number up to the nearest odd integer.
@@ -869,10 +887,12 @@ def TRUNC(value, places=0):
 def UUID():
   """
   Generate a random UUID-formatted string identifier.
+
   Since UUID() produces a different value each time it's called, it is best to use it in
   [trigger formula](formulas.md#trigger-formulas) for new records.
-  This would only calculate UUID() once and freeze the calculated value. By contrast, a regular formula
+  This would only calculate UUID() once and freeze the calculated value. By contrast, a regular
-  may get recalculated any time the document is reloaded, producing a different value for UUID() each time.
+  formula may get recalculated any time the document is reloaded, producing a different value for
+  UUID() each time.
   """
   try:
     uid = uuid.uuid4()

sandbox/grist/import_actions.py

@@ -1,5 +1,3 @@
-from collections import namedtuple
-
 from six.moves import zip
 
 import column
@@ -86,7 +84,7 @@ class ImportActions(object):
     hidden_table_rec = tables.lookupOne(tableId=hidden_table_id)
 
     # will use these to set default formulas (if column names match in src and dest table)
-    src_cols = {c.colId for c in hidden_table_rec.columns}
+    src_cols = {c.colId: c for c in hidden_table_rec.columns}
 
     target_table = tables.lookupOne(tableId=dest_table_id) if dest_table_id else hidden_table_rec
     target_cols = target_table.columns
@@ -97,34 +95,29 @@ class ImportActions(object):
     dest_cols = []
     for c in target_cols:
       if column.is_visible_column(c.colId) and (not c.isFormula or c.formula == ""):
-
+        source_col = src_cols.get(c.colId)
-        dest_cols.append( {
+        dest_col = {
           "label":    c.label,
           "colId":    c.colId if dest_table_id else None, #should be None if into new table
           "type":     c.type,
           "widgetOptions": getattr(c, "widgetOptions", ""),
-          "formula":  ("$" + c.colId) if (c.colId in src_cols) else ''
+          "formula":  ("$" + source_col.colId) if source_col else '',
-        })
+        }
+        if source_col and c.type.startswith("Ref:"):
+          ref_table_id = c.type.split(':')[1]
+          visible_col = c.visibleCol
+          if visible_col:
+            dest_col["visibleCol"] = visible_col.id
+            dest_col["formula"] = '{}.lookupOne({}=${c}) or (${c} and str(${c}))'.format(
+                ref_table_id, visible_col.colId, c=source_col.colId)
+
+        dest_cols.append(dest_col)
 
     return {"destCols": dest_cols}
     # doesnt generate other fields of transform_rule, but sandbox only used destCols
 
 
-  def FillTransformRuleColIds(self, transform_rule):
+  def _MakeImportTransformColumns(self, hidden_table_id, transform_rule, gen_all):
-    """
-    Takes a transform rule with missing dest col ids, and returns it
-    with sanitized and de-duplicated ids generated from the original
-    column labels.
-
-    NOTE: This work could be done outside the data engine, but the logic
-    for cleaning column identifiers is quite complex and currently only lives
-    in the data engine. In the future, it may be worth porting it to
-    Node to avoid an extra trip to the data engine.
-    """
-    return _gen_colids(transform_rule)
-
-
-  def MakeImportTransformColumns(self, hidden_table_id, transform_rule, gen_all):
     """
     Makes prefixed columns in the grist hidden import table (hidden_table_id)
 
@@ -142,10 +135,9 @@ class ImportActions(object):
     src_cols = {c.colId for c in hidden_table_rec.columns}
     log.debug("destCols:" + repr(transform_rule['destCols']))
 
-    #wrap dest_cols as namedtuples, to allow access like 'dest_col.param'
+    dest_cols = transform_rule['destCols']
-    dest_cols = [namedtuple('col', c.keys())(*c.values()) for c in transform_rule['destCols']]
 
-    log.debug("MakeImportTransformColumns: {}".format("gen_all" if gen_all else "optimize"))
+    log.debug("_MakeImportTransformColumns: {}".format("gen_all" if gen_all else "optimize"))
 
     # Calling rebuild_usercode once per added column is wasteful and can be very slow.
     self._engine._should_rebuild_usercode = False
@@ -156,21 +148,31 @@ class ImportActions(object):
     try:
       for c in dest_cols:
         # skip copy columns (unless gen_all)
-        formula = c.formula.strip()
+        formula = c["formula"].strip()
         isCopyFormula = (formula.startswith("$") and formula[1:] in src_cols)
 
         if gen_all or not isCopyFormula:
           # If colId specified, use that. Otherwise, use the (sanitized) label.
-          col_id = c.colId or identifiers.pick_col_ident(c.label)
+          col_id = c["colId"] or identifiers.pick_col_ident(c["label"])
+          visible_col_ref = c.get("visibleCol", 0)
           new_col_id = _import_transform_col_prefix + col_id
           new_col_spec = {
-            "label": c.label,
+            "label": c["label"],
-            "type": c.type,
+            "type": c["type"],
-            "widgetOptions": getattr(c, "widgetOptions", ""),
+            "widgetOptions": c.get("widgetOptions", ""),
             "isFormula": True,
-            "formula": c.formula}
+            "formula": c["formula"],
+            "visibleCol": visible_col_ref,
+          }
           result = self._useractions.doAddColumn(hidden_table_id, new_col_id, new_col_spec)
-          new_cols.append(result["colRef"])
+          new_col_id, new_col_ref = result["colId"], result["colRef"]
+
+          if visible_col_ref:
+            visible_col_id = self._docmodel.columns.table.get_record(visible_col_ref).colId
+            self._useractions.SetDisplayFormula(hidden_table_id, None, new_col_ref,
+                '${}.{}'.format(new_col_id, visible_col_id))
+
+          new_cols.append(new_col_ref)
     finally:
       self._engine._should_rebuild_usercode = True
     self._engine.rebuild_usercode()
@@ -178,36 +180,40 @@ class ImportActions(object):
     return new_cols
 
 
-  def DoGenImporterView(self, source_table_id, dest_table_id, transform_rule = None):
+  def DoGenImporterView(self, source_table_id, dest_table_id, transform_rule, options):
     """
-    Generates viewsections/formula columns for importer
+    Generates formula columns for transformed importer columns, and optionally a new viewsection.
 
-    source_table_id: id of temporary hidden table, data parsed from data source
+    source_table_id: id of temporary hidden table, containing source data and used for preview.
-    dest_table_id: id of table to import to, or None for new table
+    dest_table_id: id of table to import to, or None for new table.
     transform_rule: transform_rule to reuse (if it still applies), if None will generate new one
+    options: a dictionary with optional keys:
+      createViewSection: defaults to True, in which case creates a new view-section to show the
+        generated columns, for use in review, and remove any previous ones.
+      genAll: defaults to True; if False, transform formulas that just copy will not be generated.
+      refsAsInts: if set, treat Ref columns as type Int for a new dest_table. This is used when
+        finishing imports from multi-table sources (e.g. from json) to avoid issues such as
+        importing linked tables in the wrong order. Caller is expected to fix these up separately.
 
-    Removes old transform viewSection and columns for source_table_id, and creates new ones that
+    Returns and object with:
-    match the destination table.
+      transformRule: updated (normalized) transform rule, or a newly generated one.
-    Returns the rowId of the newly added section or 0 if no source table (source_table_id
+      viewSectionRef: rowId of the newly added section, present only if createViewSection is set.
-    can be None in case of importing empty file).
-
-    Creates formula columns for transforms (match columns in dest table)
     """
+    createViewSection = options.get("createViewSection", True)
+    genAll = options.get("genAll", True)
+    refsAsInts = options.get("refsAsInts", True)
 
-    tables = self._docmodel.tables
+    if createViewSection:
-    src_table_rec = tables.lookupOne(tableId=source_table_id)
+      src_table_rec = self._docmodel.tables.lookupOne(tableId=source_table_id)
 
-    # for new table, dest_table_id is None
+      # ======== Cleanup old sections/columns
-    dst_table_rec = tables.lookupOne(tableId=dest_table_id) if dest_table_id else src_table_rec
 
-    # ======== Cleanup old sections/columns
+      # Transform columns are those that start with a special prefix.
-
+      old_cols = [c for c in src_table_rec.columns
-    # Transform columns are those that start with a special prefix.
+                  if c.colId.startswith(_import_transform_col_prefix)]
-    old_cols = [c for c in src_table_rec.columns
+      old_sections = {field.parentId for c in old_cols for field in c.viewFields}
-                if c.colId.startswith(_import_transform_col_prefix)]
+      self._docmodel.remove(old_sections)
-    old_sections = {field.parentId for c in old_cols for field in c.viewFields}
+      self._docmodel.remove(old_cols)
-    self._docmodel.remove(old_sections)
-    self._docmodel.remove(old_cols)
 
     #======== Prepare/normalize transform_rule, Create new formula columns
     # Defaults to duplicating dest_table columns (or src_table columns for a new table)
@@ -216,26 +222,35 @@ class ImportActions(object):
     if transform_rule is None:
       transform_rule = self._MakeDefaultTransformRule(source_table_id, dest_table_id)
 
-    else: #ensure prefixes, colIds are correct
+    # ensure prefixes, colIds are correct
-      _strip_prefixes(transform_rule)
+    _strip_prefixes(transform_rule)
 
-      if not dest_table_id: # into new table: 'colId's are undefined
+    if not dest_table_id: # into new table: 'colId's are undefined
-        _gen_colids(transform_rule)
+      _gen_colids(transform_rule)
-      else:
-        if None in (dc["colId"] for dc in transform_rule["destCols"]):
-          errstr = "colIds must be defined in transform_rule for importing into existing table: "
-          raise ValueError(errstr + repr(transform_rule))
 
+      # Treat destination Ref:* columns as Int instead, for new tables, to avoid issues when
+      # importing linked tables in the wrong order. Caller is expected to fix up afterwards.
+      if refsAsInts:
+        for col in transform_rule["destCols"]:
+          if col["type"].startswith("Ref:"):
+            col["type"] = "Int"
 
-    new_cols = self.MakeImportTransformColumns(source_table_id, transform_rule, gen_all=True)
+    else:
-    # we want to generate all columns so user can see them and edit
+      if None in (dc["colId"] for dc in transform_rule["destCols"]):
+        errstr = "colIds must be defined in transform_rule for importing into existing table: "
+        raise ValueError(errstr + repr(transform_rule))
 
-    #=========  Create new transform view section.
+    new_cols = self._MakeImportTransformColumns(source_table_id, transform_rule, gen_all=genAll)
-    new_section = self._docmodel.add(self._docmodel.view_sections,
-                                    tableRef=src_table_rec.id,
-                                    parentKey='record',
-                                    borderWidth=1, defaultWidth=100,
-                                    sortColRefs='[]')[0]
-    self._docmodel.add(new_section.fields, colRef=new_cols)
 
-    return new_section.id
+    result = {"transformRule": transform_rule}
+    if createViewSection:
+      #=========  Create new transform view section.
+      new_section = self._docmodel.add(self._docmodel.view_sections,
+                                      tableRef=src_table_rec.id,
+                                      parentKey='record',
+                                      borderWidth=1, defaultWidth=100,
+                                      sortColRefs='[]')[0]
+      self._docmodel.add(new_section.fields, colRef=new_cols)
+      result["viewSectionRef"] = new_section.id
+
+    return result

sandbox/grist/test_import_actions.py

@@ -66,7 +66,7 @@ class TestImportActions(test_engine.EngineTestCase):
 
     # Update transform while importing to destination table which have
     # columns with the same names as source
-    self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None])
+    self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None, {}])
 
     # Verify the new structure of source table and sections
     # (two columns with special names were added)
@@ -98,7 +98,7 @@ class TestImportActions(test_engine.EngineTestCase):
 
     # Apply useraction again to verify that old columns and sections are removing
     # Update transform while importing to destination table which has no common columns with source
-    self.apply_user_action(['GenImporterView', 'Source', 'Destination2', None])
+    self.apply_user_action(['GenImporterView', 'Source', 'Destination2', None, {}])
 
     # Verify the new structure of source table and sections (old special columns were removed
     # and one new columns with empty formula were added)
@@ -131,8 +131,8 @@ class TestImportActions(test_engine.EngineTestCase):
     # Generate without a destination table, and then with one. Ensure that we don't omit the
     # actions needed to populate the table in the second call.
     self.init_state()
-    self.apply_user_action(['GenImporterView', 'Source', None, None])
+    self.apply_user_action(['GenImporterView', 'Source', None, None, {}])
-    out_actions = self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None])
+    out_actions = self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None, {}])
     self.assertPartialOutActions(out_actions, {
       "stored": [
         ["BulkRemoveRecord", "_grist_Views_section_field", [13, 14, 15]],
@@ -160,7 +160,7 @@ class TestImportActions(test_engine.EngineTestCase):
     self.init_state()
 
     # Update transform while importing to destination table which is "New Table"
-    self.apply_user_action(['GenImporterView', 'Source', None, None])
+    self.apply_user_action(['GenImporterView', 'Source', None, None, {}])
 
     # Verify the new structure of source table and sections (old special columns were removed
     # and three new columns, which are the same as in source table were added)

sandbox/grist/useractions.py

@@ -2134,13 +2134,6 @@ class UserActions(object):
   #----------------------------------------------------------------------
 
   @useraction
-  def GenImporterView(self, source_table_id, dest_table_id, transform_rule = None):
+  def GenImporterView(self, source_table_id, dest_table_id, transform_rule=None, options=None):
-    return self._import_actions.DoGenImporterView(source_table_id, dest_table_id, transform_rule)
+    return self._import_actions.DoGenImporterView(
-
+        source_table_id, dest_table_id, transform_rule, options or {})
-  @useraction
-  def MakeImportTransformColumns(self, source_table_id, transform_rule, gen_all):
-    return self._import_actions.MakeImportTransformColumns(source_table_id, transform_rule, gen_all)
-
-  @useraction
-  def FillTransformRuleColIds(self, transform_rule):
-    return self._import_actions.FillTransformRuleColIds(transform_rule)

test/fixtures/export-csv/reference.csv

@@ -1,4 +1,4 @@
-Text Bar,Int Text,Text Formula
+Text Bar,Int Text,Text Formula,RowId
-"the ""quote marks"" ?",Integer[500],a --- grist https://www.getgrist.com/
+"the ""quote marks"" ?",500,a --- grist https://www.getgrist.com/,Text[3]
-"b ,d",Integer[200],"b ,d --- https://www.getgrist.com/"
+"b ,d",200,"b ,d --- https://www.getgrist.com/",Text[2]
-"the ""quote marks"" ?",,a --- grist https://www.getgrist.com/
+"the ""quote marks"" ?",0,a --- grist https://www.getgrist.com/,

test/fixtures/uploads/ImportReferences-Tasks.csv

@@ -0,0 +1,5 @@
+Label,PName,PIndex,PIndex2,PDate,PRowID,PID
+Foo2,Clean,1000,"1,000",27 Mar 2023,,0
+Bar2,Wash,3000,"2,000",,Projects[2],2
+Baz2,Build2,,2,20 Mar 2023,Projects[1],1
+Zoo2,Clean,2000,"4,000",24 Apr 2023,Projects[3],3