(core) Fix imports into reference columns, and support two ways to import Numeric as a reference.

Summary:
- When importing into a Ref column, use lookupOne() formula for correct previews.
- When selecting columns to import into a Ref column, now a Numeric column like
  'Order' will produce two options: "Order" and "Order (as row ID)".
- Fixes exports to correct the formatting of visible columns. This addresses multiple bugs:
  1. Formatting wasn't used, e.g. a Ref showing a custom-formatted date was still presented as YYYY-MM-DD in CSVs.
  2. Ref showing a Numeric column was formatted as if a row ID (e.g. `Table1[1.5]`), which is very wrong.
- If importing into a table that doesn't have a primary view, don't switch page after import.

Refactorings:
- Generalize GenImporterView to be usable in more cases; removed near-duplicated logic from node side
- Some other refactoring in importing code.
- Fix field/column option selection in ValueParser
- Add NUM() helper to turn integer-valued floats into ints, useful for "as row ID" lookups.

Test Plan: Added test cases for imports into reference columns, updated Exports test fixtures.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3875
This commit is contained in:
Dmitry S 2023-04-25 17:11:25 -04:00
parent 7a12a8ef28
commit 65013331a3
17 changed files with 290 additions and 339 deletions

View File

@ -19,7 +19,8 @@ import {openFilePicker} from 'app/client/ui/FileDialog';
import {bigBasicButton, bigPrimaryButton} from 'app/client/ui2018/buttons';
import {testId, theme, vars} from 'app/client/ui2018/cssVars';
import {icon} from 'app/client/ui2018/icons';
import {IOptionFull, linkSelect, menu, menuDivider, menuItem, multiSelect} from 'app/client/ui2018/menus';
import {IOptionFull, linkSelect, menu,
menuDivider, menuItem, multiSelect} from 'app/client/ui2018/menus';
import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals';
import {loadingSpinner} from 'app/client/ui2018/loaders';
import {openFormulaEditor} from 'app/client/widgets/FormulaEditor';
@ -188,30 +189,29 @@ export class Importer extends DisposableWithEvents {
...use(this._gristDoc.docModel.visibleTableIds.getObservable()).map((id) => ({value: id, label: id})),
]);
// Source column labels for the selected import source, keyed by column id.
private _sourceColLabelsById = Computed.create(this, this._sourceInfoSelected, (use, info) => {
if (!info || use(info.sourceSection._isDeleted)) { return null; }
const fields = use(use(info.sourceSection.viewFields).getObservable());
return new Map(fields.map(f => [use(use(f.column).colId), use(use(f.column).label)]));
// List of transform fields, i.e. those formula fields of the transform section whose values
// will be used to populate the destination columns.
private _transformFields: Computed<ViewFieldRec[]|null> = Computed.create(
this, this._sourceInfoSelected, (use, info) => {
const section = info && use(info.transformSection);
if (!section || use(section._isDeleted)) { return null; }
return use(use(section.viewFields).getObservable());
});
// Transform section columns of the selected source.
private _transformSectionCols = Computed.create(this, this._sourceInfoSelected, (use, info) => {
if (!info) { return null; }
const transformSection = use(info.transformSection);
if (!transformSection || use(transformSection._isDeleted)) { return null; }
const fields = use(use(transformSection.viewFields).getObservable());
return fields.map(f => use(f.column));
// Prepare a Map, mapping of colRef of each transform column to the set of options to offer in
// the dropdown. The options are represented as a Map too, mapping formula to label.
private _transformColImportOptions: Computed<Map<number, Map<string, string>>> = Computed.create(
this, this._transformFields, this._sourceInfoSelected, (use, fields, info) => {
if (!fields || !info) { return new Map(); }
return new Map(fields.map(f =>
[use(f.colRef), this._makeImportOptionsForCol(use(f.column), info)]));
});
// List of destination fields that aren't mapped to a source column.
private _unmatchedFields = Computed.create(this, this._transformSectionCols, (use, cols) => {
if (!cols) { return null; }
return cols.filter(c => use(c.formula).trim() === '').map(c => c.label());
// List of labels of destination columns that aren't mapped to a source column, i.e. transform
// columns with empty formulas.
private _unmatchedFields: Computed<string[]|undefined> = Computed.create(
this, this._transformFields, (use, fields) => {
return fields?.filter(f => (use(use(f.column).formula).trim() === '')).map(f => use(f.label));
});
// null tells to use the built-in file picker.
@ -299,9 +299,9 @@ export class Importer extends DisposableWithEvents {
sourceInfo.transformSection.set(null);
const genImporterViewPromise = this._gristDoc.docData.sendAction(
['GenImporterView', sourceInfo.hiddenTableId, sourceInfo.destTableId.get(), null]);
['GenImporterView', sourceInfo.hiddenTableId, sourceInfo.destTableId.get(), null, null]);
sourceInfo.lastGenImporterViewPromise = genImporterViewPromise;
const transformSectionRef = await genImporterViewPromise;
const transformSectionRef = (await genImporterViewPromise).viewSectionRef;
// If the request is superseded by a newer request, or the Importer is disposed, do nothing.
if (this.isDisposed() || sourceInfo.lastGenImporterViewPromise !== genImporterViewPromise) {
@ -444,7 +444,11 @@ export class Importer extends DisposableWithEvents {
if (importResult.tables[0]?.hiddenTableId) {
const tableRowModel = this._gristDoc.docModel.dataTables[importResult.tables[0].hiddenTableId].tableMetaRow;
await this._gristDoc.openDocPage(tableRowModel.primaryViewId());
const primaryViewId = tableRowModel.primaryViewId();
if (primaryViewId) {
// Switch page if there is a sensible one to switch to.
await this._gristDoc.openDocPage(primaryViewId);
}
}
this._screen.close();
this.dispose();
@ -685,49 +689,14 @@ export class Importer extends DisposableWithEvents {
),
cssDestinationFieldSettings(
icon('Dots'),
menu(
() => {
const sourceColId = field.origCol().id();
const sourceColIdsAndLabels = [...this._sourceColLabelsById.get()!.entries()];
return [
menuItem(
async () => {
await this._gristDoc.clearColumns([sourceColId], {keepType: true});
await this._updateImportDiff(info);
},
'Skip',
testId('importer-column-match-menu-item')
),
menuDivider(),
...sourceColIdsAndLabels.map(([id, label]) =>
menuItem(
async () => {
await this._setColumnFormula(sourceColId, '$' + id);
await this._updateImportDiff(info);
},
label,
testId('importer-column-match-menu-item')
),
),
testId('importer-column-match-menu'),
];
},
menu(() => this._makeImportOptionsMenu(field.origCol.peek(), info),
{ placement: 'right-start' },
),
testId('importer-column-match-destination-settings')
),
testId('importer-column-match-destination')
),
dom.domComputed(use => dom.create(
this._buildColMappingFormula.bind(this),
use(field.column),
(elem: Element) => this._activateFormulaEditor(
elem,
field,
() => this._updateImportDiff(info),
),
'Skip'
)),
dom.create(owner => this._buildColMappingFormula(owner, field, info)),
testId('importer-column-match-source-destination'),
)
)),
@ -776,6 +745,52 @@ export class Importer extends DisposableWithEvents {
this._screen.render(content, {fullscreen: true});
}
private _makeImportOptionsForCol(transformCol: ColumnRec, info: SourceInfo) {
const options = new Map<string, string>(); // Maps formula to label.
const importedFields = info.sourceSection.viewFields.peek().peek();
// Reference columns are populated using lookup formulas, so figure out now if this is a
// reference column, and if so, its destination table and the lookup column ID.
const refTable = transformCol.refTable.peek();
const refTableId = refTable ? refTable.tableId.peek() : undefined;
const visibleColId = transformCol.visibleColModel.peek().colId.peek();
const isRefDest = Boolean(info.destTableId.get() && transformCol.pureType.peek() === 'Ref');
for (const f of importedFields) {
const importedCol = f.column.peek();
const colId = importedCol.colId.peek();
const colLabel = importedCol.label.peek();
if (isRefDest && visibleColId) {
const formula = `${refTableId}.lookupOne(${visibleColId}=$${colId}) or ($${colId} and str($${colId}))`;
options.set(formula, colLabel);
} else {
options.set(`$${colId}`, colLabel);
}
if (isRefDest && ['Numeric', 'Int'].includes(importedCol.type.peek())) {
options.set(`${refTableId}.lookupOne(id=NUM($${colId})) or ($${colId} and str(NUM($${colId})))`,
`${colLabel} (as row ID)`);
}
}
return options;
}
private _makeImportOptionsMenu(transformCol: ColumnRec, info: SourceInfo) {
const transformColRef = transformCol.id();
const options = this._transformColImportOptions.get().get(transformCol.getRowId());
return [
menuItem(() => this._setColumnFormula(transformColRef, null, info),
'Skip',
testId('importer-column-match-menu-item')),
menuDivider(),
...Array.from(options || [], ([formula, label]) =>
menuItem(() => this._setColumnFormula(transformColRef, formula, info),
label,
testId('importer-column-match-menu-item'))
),
testId('importer-column-match-menu'),
];
}
private _addFocusLayer(container: HTMLElement) {
dom.autoDisposeElem(container, new FocusLayer({
defaultFocusElem: container,
@ -787,10 +802,14 @@ export class Importer extends DisposableWithEvents {
/**
* Updates the formula on column `colRef` to `formula`.
*/
private async _setColumnFormula(colRef: number, formula: string): Promise<void> {
return this._gristDoc.docModel.columns.sendTableAction(
['UpdateRecord', colRef, { formula, isFormula: true }]
);
private async _setColumnFormula(transformColRef: number, formula: string|null, info: SourceInfo) {
if (formula === null) {
await this._gristDoc.clearColumns([transformColRef], {keepType: true});
} else {
await this._gristDoc.docModel.columns.sendTableAction(
['UpdateRecord', transformColRef, { formula, isFormula: true }]);
}
await this._updateImportDiff(info);
}
/**
@ -841,26 +860,20 @@ export class Importer extends DisposableWithEvents {
* in the column mapping section of Importer. On click, opens
* an editor for the formula for `column`.
*/
private _buildColMappingFormula(_owner: MultiHolder, column: ColumnRec, buildEditor: (e: Element) => void,
placeholder: string) {
const formatFormula = (formula: string) => {
const sourceColLabels = this._sourceColLabelsById.get();
if (!sourceColLabels) { return formula; }
private _buildColMappingFormula(owner: MultiHolder, field: ViewFieldRec, info: SourceInfo) {
const displayFormula = Computed.create(owner, use => {
const column = use(field.column);
const formula = use(column.formula);
const importOptions = use(this._transformColImportOptions).get(column.getRowId());
return importOptions?.get(formula) ?? formula;
});
formula = formula.trim();
if (formula.startsWith('$') && sourceColLabels.has(formula.slice(1))) {
// For simple formulas that only reference a source column id, show the source column label.
return sourceColLabels.get(formula.slice(1))!;
}
return formula;
};
return cssFieldFormula(use => formatFormula(use(column.formula)),
{gristTheme: this._gristDoc.currentTheme, placeholder, maxLines: 1},
return cssFieldFormula(displayFormula,
{gristTheme: this._gristDoc.currentTheme, placeholder: 'Skip', maxLines: 1},
dom.cls('disabled'),
{tabIndex: '-1'},
dom.on('focus', (_ev, elem) => buildEditor(elem)),
dom.on('focus', (_ev, elem) =>
this._activateFormulaEditor(elem, field, () => this._updateImportDiff(info))),
testId('importer-column-match-formula'),
);
}

View File

@ -70,7 +70,7 @@ export type IsRightTypeFunc = (value: CellValue) => boolean;
export class BaseFormatter {
protected isRightType: IsRightTypeFunc;
constructor(public type: string, public widgetOpts: object, public docSettings: DocumentSettings) {
constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) {
this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) ||
gristTypes.isRightType('Any')!;
}
@ -144,7 +144,7 @@ export interface DateFormatOptions {
}
class DateFormatter extends BaseFormatter {
private _dateTimeFormat: string;
protected _dateTimeFormat: string;
private _timezone: string;
constructor(type: string, widgetOpts: DateFormatOptions, docSettings: DocumentSettings, timezone: string = 'UTC') {
@ -194,9 +194,11 @@ export interface DateTimeFormatOptions extends DateFormatOptions {
class DateTimeFormatter extends DateFormatter {
constructor(type: string, widgetOpts: DateTimeFormatOptions, docSettings: DocumentSettings) {
const timezone = gutil.removePrefix(type, "DateTime:") || '';
// Pass up the original widgetOpts. It's helpful to have them available; e.g. ExcelFormatter
// takes options from an initialized ValueFormatter.
super(type, widgetOpts, docSettings, timezone);
const timeFormat = widgetOpts.timeFormat === undefined ? 'h:mma' : widgetOpts.timeFormat;
const dateFormat = (widgetOpts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
super(type, {dateFormat}, docSettings, timezone);
this._dateTimeFormat = (widgetOpts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
}
}

View File

@ -268,7 +268,8 @@ export function createParserOrFormatterArguments(
const col = columnsTable.getRecord(colRef)!;
let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col;
if (fieldRef) {
fieldOrCol = fieldsTable.getRecord(fieldRef) || col;
const field = fieldsTable.getRecord(fieldRef);
fieldOrCol = field?.widgetOptions ? field : col;
}
return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol);

View File

@ -5,11 +5,11 @@ import * as _ from 'underscore';
import {ColumnDelta, createEmptyActionSummary} from 'app/common/ActionSummary';
import {ApplyUAResult, DataSourceTransformed, ImportOptions, ImportResult, ImportTableResult,
MergeOptions, MergeOptionsMap, MergeStrategy, SKIP_TABLE, TransformColumn,
MergeOptions, MergeOptionsMap, MergeStrategy, SKIP_TABLE,
TransformRule,
TransformRuleMap} from 'app/common/ActiveDocAPI';
import {ApiError} from 'app/common/ApiError';
import {BulkColValues, CellValue, fromTableDataAction, TableRecordValue, UserAction} from 'app/common/DocActions';
import {BulkColValues, CellValue, fromTableDataAction, UserAction} from 'app/common/DocActions';
import * as gutil from 'app/common/gutil';
import {DocStateComparison} from 'app/common/UserAPI';
import {guessColInfoForImports} from 'app/common/ValueGuesser';
@ -339,9 +339,9 @@ export class ActiveDocImport {
if (isHidden) {
// Generate formula columns, view sections, etc
const results: ApplyUAResult = await this._activeDoc.applyUserActions(docSession,
[['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null]]);
[['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null, null]]);
transformSectionRef = results.retValues[0];
transformSectionRef = results.retValues[0].viewSectionRef;
createdTableId = hiddenTableId;
} else {
@ -391,36 +391,21 @@ export class ActiveDocImport {
* the source and destination table.
* @returns {string} The table id of the new or updated destination table.
*/
private async _transformAndFinishImport(docSession: OptDocSession,
private async _transformAndFinishImport(
docSession: OptDocSession,
hiddenTableId: string, destTableId: string,
intoNewTable: boolean, transformRule: TransformRule|null,
mergeOptions: MergeOptions|null): Promise<string> {
mergeOptions: MergeOptions|null
): Promise<string> {
log.info("ActiveDocImport._transformAndFinishImport(%s, %s, %s, %s, %s)",
hiddenTableId, destTableId, intoNewTable, transformRule, mergeOptions);
const srcCols = await this._activeDoc.getTableCols(docSession, hiddenTableId);
// Use a default transform rule if one was not provided by the client.
if (!transformRule) {
const transformDest = intoNewTable ? null : destTableId;
transformRule = await this._makeDefaultTransformRule(docSession, srcCols, transformDest);
}
// Transform rules from client may have prefixed column ids, so we need to strip them.
stripRulePrefixes(transformRule);
if (intoNewTable) {
// Transform rules for new tables don't have filled in destination column ids.
const result = await this._activeDoc.applyUserActions(docSession, [['FillTransformRuleColIds', transformRule]]);
transformRule = result.retValues[0] as TransformRule;
// Encode Refs as Ints, to avoid table dependency issues. We'll convert back to Ref at the end.
encodeRuleReferences(transformRule);
} else if (transformRule.destCols.some(c => c.colId === null)) {
throw new Error('Column ids in transform rule must be filled when importing into an existing table');
}
await this._activeDoc.applyUserActions(docSession,
[['MakeImportTransformColumns', hiddenTableId, transformRule, false]]);
const transformDestTableId = intoNewTable ? null : destTableId;
const result = await this._activeDoc.applyUserActions(docSession, [[
'GenImporterView', hiddenTableId, transformDestTableId, transformRule,
{createViewSection: false, genAll: false, refsAsInts: true},
]]);
transformRule = result.retValues[0].transformRule as TransformRule;
if (!intoNewTable && mergeOptions && mergeOptions.mergeCols.length > 0) {
await this._mergeAndFinishImport(docSession, hiddenTableId, destTableId, transformRule, mergeOptions);
@ -430,6 +415,7 @@ export class ActiveDocImport {
const hiddenTableData = fromTableDataAction(await this._activeDoc.fetchTable(docSession, hiddenTableId, true));
const columnData: BulkColValues = {};
const srcCols = await this._activeDoc.getTableCols(docSession, hiddenTableId);
const srcColIds = srcCols.map(c => c.id as string);
// Only include destination columns that weren't skipped.
@ -591,38 +577,6 @@ export class ActiveDocImport {
return this._activeDoc.docStorage.decodeMarshalledDataFromTables(result);
}
/**
* Returns a default TransformRule using column definitions from `destTableId`. If `destTableId`
* is null (in the case when the import destination is a new table), the `srcCols` are used instead.
*
* @param {TableRecordValue[]} srcCols Source column definitions.
* @param {string|null} destTableId The destination table id. If null, the destination is assumed
* to be a new table, and `srcCols` are used to build the transform rule.
* @returns {Promise<TransformRule>} The constructed transform rule.
*/
private async _makeDefaultTransformRule(docSession: OptDocSession, srcCols: TableRecordValue[],
destTableId: string|null): Promise<TransformRule> {
const targetCols = destTableId ? await this._activeDoc.getTableCols(docSession, destTableId) : srcCols;
const destCols: TransformColumn[] = [];
const srcColIds = srcCols.map(c => c.id as string);
for (const {id, fields} of targetCols) {
destCols.push({
colId: destTableId ? id as string : null,
label: fields.label as string,
type: fields.type as string,
widgetOptions: fields.widgetOptions as string,
formula: srcColIds.includes(id as string) ? `$${id}` : ''
});
}
return {
destTableId,
destCols,
sourceCols: srcColIds
};
}
/**
* This function removes temporary hidden tables which were created during the import process
*
@ -694,32 +648,6 @@ function isBlank(value: CellValue): boolean {
return value === null || (typeof value === 'string' && value.trim().length === 0);
}
/**
* Changes every Ref column to an Int column in `destCols`.
*
* Encoding references as ints can be useful when finishing imports to avoid
* issues such as importing linked tables in the wrong order. When encoding references,
* ActiveDocImport._fixReferences should be called at the end of importing to
* decode Ints back to Refs.
*/
function encodeRuleReferences({destCols}: TransformRule): void {
for (const col of destCols) {
const refTableId = gutil.removePrefix(col.type, "Ref:");
if (refTableId) {
col.type = 'Int';
}
}
}
// Helper function that strips import prefixes from columns in transform rules (if ids are present).
function stripRulePrefixes({destCols}: TransformRule): void {
for (const col of destCols) {
const colId = col.colId;
if (colId && colId.startsWith(IMPORT_TRANSFORM_COLUMN_PREFIX)) {
col.colId = colId.slice(IMPORT_TRANSFORM_COLUMN_PREFIX.length);
}
}
}
// Helper function that returns new `colIds` with import prefixes stripped.
function stripPrefixes(colIds: string[]): string[] {

View File

@ -1,6 +1,7 @@
import {ApiError} from 'app/common/ApiError';
import {buildColFilter} from 'app/common/ColumnFilterFunc';
import {TableDataAction} from 'app/common/DocActions';
import {DocData} from 'app/common/DocData';
import {DocumentSettings} from 'app/common/DocumentSettings';
import * as gristTypes from 'app/common/gristTypes';
import * as gutil from 'app/common/gutil';
@ -11,6 +12,7 @@ import {schema, SchemaTypes} from 'app/common/schema';
import {SortFunc} from 'app/common/SortFunc';
import {Sort} from 'app/common/SortSpec';
import {MetaRowRecord, MetaTableData} from 'app/common/TableData';
import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFormatter';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {RequestWithLogin} from 'app/server/lib/Authorizer';
import {docSessionFromRequest} from 'app/server/lib/DocSession';
@ -23,23 +25,16 @@ import * as _ from 'underscore';
type Access = (row: number) => any;
// Helper interface with information about the column
interface ExportColumn {
export interface ExportColumn {
id: number;
colId: string;
label: string;
type: string;
widgetOptions: any;
formatter: BaseFormatter;
parentPos: number;
description: string;
}
// helper for empty column
const emptyCol: ExportColumn = {
id: 0,
colId: '',
label: '',
type: '',
widgetOptions: null,
parentPos: 0
};
/**
* Bare data that is exported - used to convert to various formats.
*/
@ -173,41 +168,33 @@ export async function exportTable(
{metaTables}: {metaTables?: FilteredMetaTables} = {},
): Promise<ExportData> {
metaTables = metaTables || await getMetaTables(activeDoc, req);
const docData = new DocData((tableId) => { throw new Error("Unexpected DocData fetch"); }, metaTables);
const tables = safeTable(metaTables, '_grist_Tables');
const metaColumns = safeTable(metaTables, '_grist_Tables_column');
checkTableAccess(tables, tableRef);
const table = safeRecord(tables, tableRef);
const tableColumns = safeTable(metaTables, '_grist_Tables_column')
.getRecords()
// Select only columns that belong to this table.
const tableColumns = metaColumns.filterRecords({parentId: tableRef})
// sort by parentPos and id, which should be the same order as in raw data
.sort((c1, c2) => nativeCompare(c1.parentPos, c2.parentPos) || nativeCompare(c1.id, c2.id))
// remove manual sort column
.filter(col => col.colId !== gristTypes.MANUALSORT);
.sort((c1, c2) => nativeCompare(c1.parentPos, c2.parentPos) || nativeCompare(c1.id, c2.id));
// Produce a column description matching what user will see / expect to export
const tableColsById = _.indexBy(tableColumns, 'id');
const columns = tableColumns.map(tc => {
// remove all columns that don't belong to this table
if (tc.parentId !== tableRef) {
return emptyCol;
}
// remove all helpers
if (gristTypes.isHiddenCol(tc.colId)) {
return emptyCol;
}
const columns: ExportColumn[] = tableColumns
.filter(tc => !gristTypes.isHiddenCol(tc.colId)) // Exclude helpers
.map<ExportColumn>(tc => {
// for reference columns, return display column, and copy settings from visible column
const displayCol = tableColsById[tc.displayCol || tc.id];
const colOptions = gutil.safeJsonParse(tc.widgetOptions, {});
const displayOptions = gutil.safeJsonParse(displayCol.widgetOptions, {});
const widgetOptions = Object.assign(displayOptions, colOptions);
const displayCol = metaColumns.getRecord(tc.displayCol) || tc;
return {
id: displayCol.id,
colId: displayCol.colId,
label: tc.label,
type: displayCol.type,
widgetOptions,
type: tc.type,
formatter: createFullFormatterFromDocData(docData, tc.id),
parentPos: tc.parentPos,
description: displayCol.description,
description: tc.description,
};
}).filter(tc => tc !== emptyCol);
});
// fetch actual data
const {tableData} = await activeDoc.fetchTable(docSessionFromRequest(req as RequestWithLogin), table.tableId, true);
@ -251,37 +238,35 @@ export async function exportSection(
{metaTables}: {metaTables?: FilteredMetaTables} = {},
): Promise<ExportData> {
metaTables = metaTables || await getMetaTables(activeDoc, req);
const docData = new DocData((tableId) => { throw new Error("Unexpected DocData fetch"); }, metaTables);
const viewSections = safeTable(metaTables, '_grist_Views_section');
const viewSection = safeRecord(viewSections, viewSectionId);
safe(viewSection.tableRef, `Cannot find or access table`);
const tables = safeTable(metaTables, '_grist_Tables');
checkTableAccess(tables, viewSection.tableRef);
const table = safeRecord(tables, viewSection.tableRef);
const columns = safeTable(metaTables, '_grist_Tables_column')
.filterRecords({parentId: table.id});
const metaColumns = safeTable(metaTables, '_grist_Tables_column');
const columns = metaColumns.filterRecords({parentId: table.id});
const viewSectionFields = safeTable(metaTables, '_grist_Views_section_field');
const fields = viewSectionFields.filterRecords({parentId: viewSection.id});
const savedFilters = safeTable(metaTables, '_grist_Filters')
.filterRecords({viewSectionRef: viewSection.id});
const tableColsById = _.indexBy(columns, 'id');
const fieldsByColRef = _.indexBy(fields, 'colRef');
const savedFiltersByColRef = _.indexBy(savedFilters, 'colRef');
const unsavedFiltersByColRef = _.indexBy(filters ?? [], 'colRef');
// Produce a column description matching what user will see / expect to export
const viewify = (col: GristTablesColumn, field?: GristViewsSectionField) => {
const displayCol = tableColsById[field?.displayCol || col.displayCol || col.id];
const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {});
const fieldWidgetOptions = field ? gutil.safeJsonParse(field.widgetOptions, {}) : {};
const viewify = (col: GristTablesColumn, field?: GristViewsSectionField): ExportColumn => {
const displayCol = metaColumns.getRecord(field?.displayCol || col.displayCol) || col;
return {
id: displayCol.id,
colId: displayCol.colId,
label: col.label,
type: col.type,
formatter: createFullFormatterFromDocData(docData, col.id, field?.id),
parentPos: col.parentPos,
description: col.description,
widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions),
};
};
const buildFilters = (col: GristTablesColumn, field?: GristViewsSectionField) => {
@ -297,14 +282,14 @@ export async function exportSection(
const columnsForFilters = columns
.filter(column => !gristTypes.isHiddenCol(column.colId))
.map(column => buildFilters(column, fieldsByColRef[column.id]));
const viewColumns = _.sortBy(fields, 'parentPos')
.map((field) => viewify(tableColsById[field.colRef], field));
const viewColumns: ExportColumn[] = _.sortBy(fields, 'parentPos')
.map((field) => viewify(metaColumns.getRecord(field.colRef)!, field));
// The columns named in sort order need to now become display columns
sortSpec = sortSpec || gutil.safeJsonParse(viewSection.sortColRefs, []);
sortSpec = sortSpec!.map((colSpec) => {
const colRef = Sort.getColRef(colSpec);
const col = tableColsById[colRef];
const col = metaColumns.getRecord(colRef);
if (!col) {
return 0;
}

View File

@ -1,5 +1,4 @@
import {ApiError} from 'app/common/ApiError';
import {createFormatter} from 'app/common/ValueFormatter';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {DownloadOptions, ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export';
import log from 'app/server/lib/log';
@ -86,7 +85,7 @@ function convertToCsv({
}: ExportData) {
// create formatters for columns
const formatters = viewColumns.map(col => createFormatter(col.type, col.widgetOptions, docSettings));
const formatters = viewColumns.map(col => col.formatter);
// Arrange the data into a row-indexed matrix, starting with column headers.
const csvMatrix = [viewColumns.map(col => col.label)];
// populate all the rows with values as strings

View File

@ -1,18 +1,7 @@
import * as express from 'express';
import {ApiError} from 'app/common/ApiError';
import {WidgetOptions} from 'app/common/WidgetOptions';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {DownloadOptions, exportTable} from 'app/server/lib/Export';
interface ExportColumn {
id: number;
colId: string;
label: string;
type: string;
widgetOptions: WidgetOptions;
description?: string;
parentPos: number;
}
import {DownloadOptions, ExportColumn, exportTable} from 'app/server/lib/Export';
interface FrictionlessFormat {
name: string;
@ -86,35 +75,36 @@ function columnsToTableSchema(
function buildTypeField(col: ExportColumn, locale: string) {
const type = col.type.split(':', 1)[0];
const widgetOptions = col.formatter.widgetOpts;
switch (type) {
case 'Text':
return {
type: 'string',
format: col.widgetOptions.widget === 'HyperLink' ? 'uri' : 'default',
format: widgetOptions.widget === 'HyperLink' ? 'uri' : 'default',
};
case 'Numeric':
return {
type: 'number',
bareNumber: col.widgetOptions?.numMode === 'decimal',
bareNumber: widgetOptions?.numMode === 'decimal',
...getNumberSeparators(locale),
};
case 'Integer':
return {
type: 'integer',
bareNumber: col.widgetOptions?.numMode === 'decimal',
bareNumber: widgetOptions?.numMode === 'decimal',
groupChar: getNumberSeparators(locale).groupChar,
};
case 'Date':
return {
type: 'date',
format: 'any',
gristFormat: col.widgetOptions?.dateFormat || 'YYYY-MM-DD',
gristFormat: widgetOptions?.dateFormat || 'YYYY-MM-DD',
};
case 'DateTime':
return {
type: 'datetime',
format: 'any',
gristFormat: `${col.widgetOptions?.dateFormat} ${col.widgetOptions?.timeFormat}`,
gristFormat: `${widgetOptions?.dateFormat} ${widgetOptions?.timeFormat}`,
};
case 'Bool':
return {
@ -125,12 +115,12 @@ function buildTypeField(col: ExportColumn, locale: string) {
case 'Choice':
return {
type: 'string',
constraints: {enum: col.widgetOptions?.choices},
constraints: {enum: widgetOptions?.choices},
};
case 'ChoiceList':
return {
type: 'array',
constraints: {enum: col.widgetOptions?.choices},
constraints: {enum: widgetOptions?.choices},
};
case 'Reference':
return {type: 'string'};

View File

@ -130,7 +130,7 @@ async function convertToExcel(tables: ExportData[], testDates: boolean) {
const { columns, rowIds, access, tableName } = table;
const ws = wb.addWorksheet(sanitizeWorksheetName(tableName));
// Build excel formatters.
const formatters = columns.map(col => createExcelFormatter(col.type, col.widgetOptions));
const formatters = columns.map(col => createExcelFormatter(col.formatter.type, col.formatter.widgetOpts));
// Generate headers for all columns with correct styles for whole column.
// Actual header style for a first row will be overwritten later.
ws.columns = columns.map((col, c) => ({ header: col.label, style: formatters[c].style() }));

View File

@ -5,10 +5,9 @@ from __future__ import absolute_import
import datetime
import math as _math
import operator
import os
import random
import uuid
from functools import reduce
from functools import reduce # pylint: disable=redefined-builtin
from six.moves import zip, xrange
import six
@ -491,6 +490,25 @@ def MULTINOMIAL(value1, *more_values):
res *= COMBIN(s, v)
return res
def NUM(value):
"""
For a Python floating-point value that's actually an integer, returns a Python integer type.
Otherwise, returns the value unchanged. This is helpful sometimes when a value comes from a
Numeric Grist column (represented as floats), but when int values are actually expected.
>>> NUM(-17.0)
-17
>>> NUM(1.5)
1.5
>>> NUM(4)
4
>>> NUM("NA")
'NA'
"""
if isinstance(value, float) and value.is_integer():
return int(value)
return value
def ODD(value):
"""
Rounds a number up to the nearest odd integer.
@ -869,10 +887,12 @@ def TRUNC(value, places=0):
def UUID():
"""
Generate a random UUID-formatted string identifier.
Since UUID() produces a different value each time it's called, it is best to use it in
[trigger formula](formulas.md#trigger-formulas) for new records.
This would only calculate UUID() once and freeze the calculated value. By contrast, a regular formula
may get recalculated any time the document is reloaded, producing a different value for UUID() each time.
This would only calculate UUID() once and freeze the calculated value. By contrast, a regular
formula may get recalculated any time the document is reloaded, producing a different value for
UUID() each time.
"""
try:
uid = uuid.uuid4()

View File

@ -1,5 +1,3 @@
from collections import namedtuple
from six.moves import zip
import column
@ -86,7 +84,7 @@ class ImportActions(object):
hidden_table_rec = tables.lookupOne(tableId=hidden_table_id)
# will use these to set default formulas (if column names match in src and dest table)
src_cols = {c.colId for c in hidden_table_rec.columns}
src_cols = {c.colId: c for c in hidden_table_rec.columns}
target_table = tables.lookupOne(tableId=dest_table_id) if dest_table_id else hidden_table_rec
target_cols = target_table.columns
@ -97,34 +95,29 @@ class ImportActions(object):
dest_cols = []
for c in target_cols:
if column.is_visible_column(c.colId) and (not c.isFormula or c.formula == ""):
dest_cols.append( {
source_col = src_cols.get(c.colId)
dest_col = {
"label": c.label,
"colId": c.colId if dest_table_id else None, #should be None if into new table
"type": c.type,
"widgetOptions": getattr(c, "widgetOptions", ""),
"formula": ("$" + c.colId) if (c.colId in src_cols) else ''
})
"formula": ("$" + source_col.colId) if source_col else '',
}
if source_col and c.type.startswith("Ref:"):
ref_table_id = c.type.split(':')[1]
visible_col = c.visibleCol
if visible_col:
dest_col["visibleCol"] = visible_col.id
dest_col["formula"] = '{}.lookupOne({}=${c}) or (${c} and str(${c}))'.format(
ref_table_id, visible_col.colId, c=source_col.colId)
dest_cols.append(dest_col)
return {"destCols": dest_cols}
# doesnt generate other fields of transform_rule, but sandbox only used destCols
def FillTransformRuleColIds(self, transform_rule):
"""
Takes a transform rule with missing dest col ids, and returns it
with sanitized and de-duplicated ids generated from the original
column labels.
NOTE: This work could be done outside the data engine, but the logic
for cleaning column identifiers is quite complex and currently only lives
in the data engine. In the future, it may be worth porting it to
Node to avoid an extra trip to the data engine.
"""
return _gen_colids(transform_rule)
def MakeImportTransformColumns(self, hidden_table_id, transform_rule, gen_all):
def _MakeImportTransformColumns(self, hidden_table_id, transform_rule, gen_all):
"""
Makes prefixed columns in the grist hidden import table (hidden_table_id)
@ -142,10 +135,9 @@ class ImportActions(object):
src_cols = {c.colId for c in hidden_table_rec.columns}
log.debug("destCols:" + repr(transform_rule['destCols']))
#wrap dest_cols as namedtuples, to allow access like 'dest_col.param'
dest_cols = [namedtuple('col', c.keys())(*c.values()) for c in transform_rule['destCols']]
dest_cols = transform_rule['destCols']
log.debug("MakeImportTransformColumns: {}".format("gen_all" if gen_all else "optimize"))
log.debug("_MakeImportTransformColumns: {}".format("gen_all" if gen_all else "optimize"))
# Calling rebuild_usercode once per added column is wasteful and can be very slow.
self._engine._should_rebuild_usercode = False
@ -156,21 +148,31 @@ class ImportActions(object):
try:
for c in dest_cols:
# skip copy columns (unless gen_all)
formula = c.formula.strip()
formula = c["formula"].strip()
isCopyFormula = (formula.startswith("$") and formula[1:] in src_cols)
if gen_all or not isCopyFormula:
# If colId specified, use that. Otherwise, use the (sanitized) label.
col_id = c.colId or identifiers.pick_col_ident(c.label)
col_id = c["colId"] or identifiers.pick_col_ident(c["label"])
visible_col_ref = c.get("visibleCol", 0)
new_col_id = _import_transform_col_prefix + col_id
new_col_spec = {
"label": c.label,
"type": c.type,
"widgetOptions": getattr(c, "widgetOptions", ""),
"label": c["label"],
"type": c["type"],
"widgetOptions": c.get("widgetOptions", ""),
"isFormula": True,
"formula": c.formula}
"formula": c["formula"],
"visibleCol": visible_col_ref,
}
result = self._useractions.doAddColumn(hidden_table_id, new_col_id, new_col_spec)
new_cols.append(result["colRef"])
new_col_id, new_col_ref = result["colId"], result["colRef"]
if visible_col_ref:
visible_col_id = self._docmodel.columns.table.get_record(visible_col_ref).colId
self._useractions.SetDisplayFormula(hidden_table_id, None, new_col_ref,
'${}.{}'.format(new_col_id, visible_col_id))
new_cols.append(new_col_ref)
finally:
self._engine._should_rebuild_usercode = True
self._engine.rebuild_usercode()
@ -178,27 +180,31 @@ class ImportActions(object):
return new_cols
def DoGenImporterView(self, source_table_id, dest_table_id, transform_rule = None):
def DoGenImporterView(self, source_table_id, dest_table_id, transform_rule, options):
"""
Generates viewsections/formula columns for importer
Generates formula columns for transformed importer columns, and optionally a new viewsection.
source_table_id: id of temporary hidden table, data parsed from data source
dest_table_id: id of table to import to, or None for new table
source_table_id: id of temporary hidden table, containing source data and used for preview.
dest_table_id: id of table to import to, or None for new table.
transform_rule: transform_rule to reuse (if it still applies), if None will generate new one
options: a dictionary with optional keys:
createViewSection: defaults to True, in which case creates a new view-section to show the
generated columns, for use in review, and remove any previous ones.
genAll: defaults to True; if False, transform formulas that just copy will not be generated.
refsAsInts: if set, treat Ref columns as type Int for a new dest_table. This is used when
finishing imports from multi-table sources (e.g. from json) to avoid issues such as
importing linked tables in the wrong order. Caller is expected to fix these up separately.
Removes old transform viewSection and columns for source_table_id, and creates new ones that
match the destination table.
Returns the rowId of the newly added section or 0 if no source table (source_table_id
can be None in case of importing empty file).
Creates formula columns for transforms (match columns in dest table)
Returns and object with:
transformRule: updated (normalized) transform rule, or a newly generated one.
viewSectionRef: rowId of the newly added section, present only if createViewSection is set.
"""
createViewSection = options.get("createViewSection", True)
genAll = options.get("genAll", True)
refsAsInts = options.get("refsAsInts", True)
tables = self._docmodel.tables
src_table_rec = tables.lookupOne(tableId=source_table_id)
# for new table, dest_table_id is None
dst_table_rec = tables.lookupOne(tableId=dest_table_id) if dest_table_id else src_table_rec
if createViewSection:
src_table_rec = self._docmodel.tables.lookupOne(tableId=source_table_id)
# ======== Cleanup old sections/columns
@ -216,20 +222,28 @@ class ImportActions(object):
if transform_rule is None:
transform_rule = self._MakeDefaultTransformRule(source_table_id, dest_table_id)
else: #ensure prefixes, colIds are correct
# ensure prefixes, colIds are correct
_strip_prefixes(transform_rule)
if not dest_table_id: # into new table: 'colId's are undefined
_gen_colids(transform_rule)
# Treat destination Ref:* columns as Int instead, for new tables, to avoid issues when
# importing linked tables in the wrong order. Caller is expected to fix up afterwards.
if refsAsInts:
for col in transform_rule["destCols"]:
if col["type"].startswith("Ref:"):
col["type"] = "Int"
else:
if None in (dc["colId"] for dc in transform_rule["destCols"]):
errstr = "colIds must be defined in transform_rule for importing into existing table: "
raise ValueError(errstr + repr(transform_rule))
new_cols = self._MakeImportTransformColumns(source_table_id, transform_rule, gen_all=genAll)
new_cols = self.MakeImportTransformColumns(source_table_id, transform_rule, gen_all=True)
# we want to generate all columns so user can see them and edit
result = {"transformRule": transform_rule}
if createViewSection:
#========= Create new transform view section.
new_section = self._docmodel.add(self._docmodel.view_sections,
tableRef=src_table_rec.id,
@ -237,5 +251,6 @@ class ImportActions(object):
borderWidth=1, defaultWidth=100,
sortColRefs='[]')[0]
self._docmodel.add(new_section.fields, colRef=new_cols)
result["viewSectionRef"] = new_section.id
return new_section.id
return result

View File

@ -66,7 +66,7 @@ class TestImportActions(test_engine.EngineTestCase):
# Update transform while importing to destination table which have
# columns with the same names as source
self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None])
self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None, {}])
# Verify the new structure of source table and sections
# (two columns with special names were added)
@ -98,7 +98,7 @@ class TestImportActions(test_engine.EngineTestCase):
# Apply useraction again to verify that old columns and sections are removing
# Update transform while importing to destination table which has no common columns with source
self.apply_user_action(['GenImporterView', 'Source', 'Destination2', None])
self.apply_user_action(['GenImporterView', 'Source', 'Destination2', None, {}])
# Verify the new structure of source table and sections (old special columns were removed
# and one new columns with empty formula were added)
@ -131,8 +131,8 @@ class TestImportActions(test_engine.EngineTestCase):
# Generate without a destination table, and then with one. Ensure that we don't omit the
# actions needed to populate the table in the second call.
self.init_state()
self.apply_user_action(['GenImporterView', 'Source', None, None])
out_actions = self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None])
self.apply_user_action(['GenImporterView', 'Source', None, None, {}])
out_actions = self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None, {}])
self.assertPartialOutActions(out_actions, {
"stored": [
["BulkRemoveRecord", "_grist_Views_section_field", [13, 14, 15]],
@ -160,7 +160,7 @@ class TestImportActions(test_engine.EngineTestCase):
self.init_state()
# Update transform while importing to destination table which is "New Table"
self.apply_user_action(['GenImporterView', 'Source', None, None])
self.apply_user_action(['GenImporterView', 'Source', None, None, {}])
# Verify the new structure of source table and sections (old special columns were removed
# and three new columns, which are the same as in source table were added)

View File

@ -2134,13 +2134,6 @@ class UserActions(object):
#----------------------------------------------------------------------
@useraction
def GenImporterView(self, source_table_id, dest_table_id, transform_rule = None):
return self._import_actions.DoGenImporterView(source_table_id, dest_table_id, transform_rule)
@useraction
def MakeImportTransformColumns(self, source_table_id, transform_rule, gen_all):
return self._import_actions.MakeImportTransformColumns(source_table_id, transform_rule, gen_all)
@useraction
def FillTransformRuleColIds(self, transform_rule):
return self._import_actions.FillTransformRuleColIds(transform_rule)
def GenImporterView(self, source_table_id, dest_table_id, transform_rule=None, options=None):
return self._import_actions.DoGenImporterView(
source_table_id, dest_table_id, transform_rule, options or {})

Binary file not shown.

Binary file not shown.

View File

@ -1,4 +1,4 @@
Text Bar,Int Text,Text Formula
"the ""quote marks"" ?",Integer[500],a --- grist https://www.getgrist.com/
"b ,d",Integer[200],"b ,d --- https://www.getgrist.com/"
"the ""quote marks"" ?",,a --- grist https://www.getgrist.com/
Text Bar,Int Text,Text Formula,RowId
"the ""quote marks"" ?",500,a --- grist https://www.getgrist.com/,Text[3]
"b ,d",200,"b ,d --- https://www.getgrist.com/",Text[2]
"the ""quote marks"" ?",0,a --- grist https://www.getgrist.com/,

1 Text Bar Int Text Text Formula RowId
2 the "quote marks" ? Integer[500] 500 a --- grist https://www.getgrist.com/ Text[3]
3 b ,d Integer[200] 200 b ,d --- https://www.getgrist.com/ Text[2]
4 the "quote marks" ? 0 a --- grist https://www.getgrist.com/

Binary file not shown.

View File

@ -0,0 +1,5 @@
Label,PName,PIndex,PIndex2,PDate,PRowID,PID
Foo2,Clean,1000,"1,000",27 Mar 2023,,0
Bar2,Wash,3000,"2,000",,Projects[2],2
Baz2,Build2,,2,20 Mar 2023,Projects[1],1
Zoo2,Clean,2000,"4,000",24 Apr 2023,Projects[3],3
1 Label PName PIndex PIndex2 PDate PRowID PID
2 Foo2 Clean 1000 1,000 27 Mar 2023 0
3 Bar2 Wash 3000 2,000 Projects[2] 2
4 Baz2 Build2 2 20 Mar 2023 Projects[1] 1
5 Zoo2 Clean 2000 4,000 24 Apr 2023 Projects[3] 3