mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) Fix imports into reference columns, and support two ways to import Numeric as a reference.
Summary: - When importing into a Ref column, use lookupOne() formula for correct previews. - When selecting columns to import into a Ref column, now a Numeric column like 'Order' will produce two options: "Order" and "Order (as row ID)". - Fixes exports to correct the formatting of visible columns. This addresses multiple bugs: 1. Formatting wasn't used, e.g. a Ref showing a custom-formatted date was still presented as YYYY-MM-DD in CSVs. 2. Ref showing a Numeric column was formatted as if a row ID (e.g. `Table1[1.5]`), which is very wrong. - If importing into a table that doesn't have a primary view, don't switch page after import. Refactorings: - Generalize GenImporterView to be usable in more cases; removed near-duplicated logic from node side - Some other refactoring in importing code. - Fix field/column option selection in ValueParser - Add NUM() helper to turn integer-valued floats into ints, useful for "as row ID" lookups. Test Plan: Added test cases for imports into reference columns, updated Exports test fixtures. Reviewers: georgegevoian Reviewed By: georgegevoian Differential Revision: https://phab.getgrist.com/D3875
This commit is contained in:
Dmitry S
parent
7a12a8ef28
commit
65013331a3
@ -19,7 +19,8 @@ import {openFilePicker} from 'app/client/ui/FileDialog';
|
||||
import {bigBasicButton, bigPrimaryButton} from 'app/client/ui2018/buttons';
|
||||
import {testId, theme, vars} from 'app/client/ui2018/cssVars';
|
||||
import {icon} from 'app/client/ui2018/icons';
|
||||
import {IOptionFull, linkSelect, menu, menuDivider, menuItem, multiSelect} from 'app/client/ui2018/menus';
|
||||
import {IOptionFull, linkSelect, menu,
|
||||
menuDivider, menuItem, multiSelect} from 'app/client/ui2018/menus';
|
||||
import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals';
|
||||
import {loadingSpinner} from 'app/client/ui2018/loaders';
|
||||
import {openFormulaEditor} from 'app/client/widgets/FormulaEditor';
|
||||
@ -188,30 +189,29 @@ export class Importer extends DisposableWithEvents {
|
||||
...use(this._gristDoc.docModel.visibleTableIds.getObservable()).map((id) => ({value: id, label: id})),
|
||||
]);
|
||||
|
||||
// Source column labels for the selected import source, keyed by column id.
|
||||
private _sourceColLabelsById = Computed.create(this, this._sourceInfoSelected, (use, info) => {
|
||||
if (!info || use(info.sourceSection._isDeleted)) { return null; }
|
||||
|
||||
const fields = use(use(info.sourceSection.viewFields).getObservable());
|
||||
return new Map(fields.map(f => [use(use(f.column).colId), use(use(f.column).label)]));
|
||||
// List of transform fields, i.e. those formula fields of the transform section whose values
|
||||
// will be used to populate the destination columns.
|
||||
private _transformFields: Computed<ViewFieldRec[]|null> = Computed.create(
|
||||
this, this._sourceInfoSelected, (use, info) => {
|
||||
const section = info && use(info.transformSection);
|
||||
if (!section || use(section._isDeleted)) { return null; }
|
||||
return use(use(section.viewFields).getObservable());
|
||||
});
|
||||
|
||||
// Transform section columns of the selected source.
|
||||
private _transformSectionCols = Computed.create(this, this._sourceInfoSelected, (use, info) => {
|
||||
if (!info) { return null; }
|
||||
|
||||
const transformSection = use(info.transformSection);
|
||||
if (!transformSection || use(transformSection._isDeleted)) { return null; }
|
||||
|
||||
const fields = use(use(transformSection.viewFields).getObservable());
|
||||
return fields.map(f => use(f.column));
|
||||
// Prepare a Map, mapping of colRef of each transform column to the set of options to offer in
|
||||
// the dropdown. The options are represented as a Map too, mapping formula to label.
|
||||
private _transformColImportOptions: Computed<Map<number, Map<string, string>>> = Computed.create(
|
||||
this, this._transformFields, this._sourceInfoSelected, (use, fields, info) => {
|
||||
if (!fields || !info) { return new Map(); }
|
||||
return new Map(fields.map(f =>
|
||||
[use(f.colRef), this._makeImportOptionsForCol(use(f.column), info)]));
|
||||
});
|
||||
|
||||
// List of destination fields that aren't mapped to a source column.
|
||||
private _unmatchedFields = Computed.create(this, this._transformSectionCols, (use, cols) => {
|
||||
if (!cols) { return null; }
|
||||
|
||||
return cols.filter(c => use(c.formula).trim() === '').map(c => c.label());
|
||||
// List of labels of destination columns that aren't mapped to a source column, i.e. transform
|
||||
// columns with empty formulas.
|
||||
private _unmatchedFields: Computed<string[]|undefined> = Computed.create(
|
||||
this, this._transformFields, (use, fields) => {
|
||||
return fields?.filter(f => (use(use(f.column).formula).trim() === '')).map(f => use(f.label));
|
||||
});
|
||||
|
||||
// null tells to use the built-in file picker.
|
||||
@ -299,9 +299,9 @@ export class Importer extends DisposableWithEvents {
|
||||
sourceInfo.transformSection.set(null);
|
||||
|
||||
const genImporterViewPromise = this._gristDoc.docData.sendAction(
|
||||
['GenImporterView', sourceInfo.hiddenTableId, sourceInfo.destTableId.get(), null]);
|
||||
['GenImporterView', sourceInfo.hiddenTableId, sourceInfo.destTableId.get(), null, null]);
|
||||
sourceInfo.lastGenImporterViewPromise = genImporterViewPromise;
|
||||
const transformSectionRef = await genImporterViewPromise;
|
||||
const transformSectionRef = (await genImporterViewPromise).viewSectionRef;
|
||||
|
||||
// If the request is superseded by a newer request, or the Importer is disposed, do nothing.
|
||||
if (this.isDisposed() || sourceInfo.lastGenImporterViewPromise !== genImporterViewPromise) {
|
||||
@ -444,7 +444,11 @@ export class Importer extends DisposableWithEvents {
|
||||
|
||||
if (importResult.tables[0]?.hiddenTableId) {
|
||||
const tableRowModel = this._gristDoc.docModel.dataTables[importResult.tables[0].hiddenTableId].tableMetaRow;
|
||||
await this._gristDoc.openDocPage(tableRowModel.primaryViewId());
|
||||
const primaryViewId = tableRowModel.primaryViewId();
|
||||
if (primaryViewId) {
|
||||
// Switch page if there is a sensible one to switch to.
|
||||
await this._gristDoc.openDocPage(primaryViewId);
|
||||
}
|
||||
}
|
||||
this._screen.close();
|
||||
this.dispose();
|
||||
@ -685,49 +689,14 @@ export class Importer extends DisposableWithEvents {
|
||||
),
|
||||
cssDestinationFieldSettings(
|
||||
icon('Dots'),
|
||||
menu(
|
||||
() => {
|
||||
const sourceColId = field.origCol().id();
|
||||
const sourceColIdsAndLabels = [...this._sourceColLabelsById.get()!.entries()];
|
||||
return [
|
||||
menuItem(
|
||||
async () => {
|
||||
await this._gristDoc.clearColumns([sourceColId], {keepType: true});
|
||||
await this._updateImportDiff(info);
|
||||
},
|
||||
'Skip',
|
||||
testId('importer-column-match-menu-item')
|
||||
),
|
||||
menuDivider(),
|
||||
...sourceColIdsAndLabels.map(([id, label]) =>
|
||||
menuItem(
|
||||
async () => {
|
||||
await this._setColumnFormula(sourceColId, '$' + id);
|
||||
await this._updateImportDiff(info);
|
||||
},
|
||||
label,
|
||||
testId('importer-column-match-menu-item')
|
||||
),
|
||||
),
|
||||
testId('importer-column-match-menu'),
|
||||
];
|
||||
},
|
||||
menu(() => this._makeImportOptionsMenu(field.origCol.peek(), info),
|
||||
{ placement: 'right-start' },
|
||||
),
|
||||
testId('importer-column-match-destination-settings')
|
||||
),
|
||||
testId('importer-column-match-destination')
|
||||
),
|
||||
dom.domComputed(use => dom.create(
|
||||
this._buildColMappingFormula.bind(this),
|
||||
use(field.column),
|
||||
(elem: Element) => this._activateFormulaEditor(
|
||||
elem,
|
||||
field,
|
||||
() => this._updateImportDiff(info),
|
||||
),
|
||||
'Skip'
|
||||
)),
|
||||
dom.create(owner => this._buildColMappingFormula(owner, field, info)),
|
||||
testId('importer-column-match-source-destination'),
|
||||
)
|
||||
)),
|
||||
@ -776,6 +745,52 @@ export class Importer extends DisposableWithEvents {
|
||||
this._screen.render(content, {fullscreen: true});
|
||||
}
|
||||
|
||||
private _makeImportOptionsForCol(transformCol: ColumnRec, info: SourceInfo) {
|
||||
const options = new Map<string, string>(); // Maps formula to label.
|
||||
const importedFields = info.sourceSection.viewFields.peek().peek();
|
||||
|
||||
// Reference columns are populated using lookup formulas, so figure out now if this is a
|
||||
// reference column, and if so, its destination table and the lookup column ID.
|
||||
const refTable = transformCol.refTable.peek();
|
||||
const refTableId = refTable ? refTable.tableId.peek() : undefined;
|
||||
const visibleColId = transformCol.visibleColModel.peek().colId.peek();
|
||||
const isRefDest = Boolean(info.destTableId.get() && transformCol.pureType.peek() === 'Ref');
|
||||
|
||||
for (const f of importedFields) {
|
||||
const importedCol = f.column.peek();
|
||||
const colId = importedCol.colId.peek();
|
||||
const colLabel = importedCol.label.peek();
|
||||
if (isRefDest && visibleColId) {
|
||||
const formula = `${refTableId}.lookupOne(${visibleColId}=$${colId}) or ($${colId} and str($${colId}))`;
|
||||
options.set(formula, colLabel);
|
||||
} else {
|
||||
options.set(`$${colId}`, colLabel);
|
||||
}
|
||||
if (isRefDest && ['Numeric', 'Int'].includes(importedCol.type.peek())) {
|
||||
options.set(`${refTableId}.lookupOne(id=NUM($${colId})) or ($${colId} and str(NUM($${colId})))`,
|
||||
`${colLabel} (as row ID)`);
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
private _makeImportOptionsMenu(transformCol: ColumnRec, info: SourceInfo) {
|
||||
const transformColRef = transformCol.id();
|
||||
const options = this._transformColImportOptions.get().get(transformCol.getRowId());
|
||||
return [
|
||||
menuItem(() => this._setColumnFormula(transformColRef, null, info),
|
||||
'Skip',
|
||||
testId('importer-column-match-menu-item')),
|
||||
menuDivider(),
|
||||
...Array.from(options || [], ([formula, label]) =>
|
||||
menuItem(() => this._setColumnFormula(transformColRef, formula, info),
|
||||
label,
|
||||
testId('importer-column-match-menu-item'))
|
||||
),
|
||||
testId('importer-column-match-menu'),
|
||||
];
|
||||
}
|
||||
|
||||
private _addFocusLayer(container: HTMLElement) {
|
||||
dom.autoDisposeElem(container, new FocusLayer({
|
||||
defaultFocusElem: container,
|
||||
@ -787,10 +802,14 @@ export class Importer extends DisposableWithEvents {
|
||||
/**
|
||||
* Updates the formula on column `colRef` to `formula`.
|
||||
*/
|
||||
private async _setColumnFormula(colRef: number, formula: string): Promise<void> {
|
||||
return this._gristDoc.docModel.columns.sendTableAction(
|
||||
['UpdateRecord', colRef, { formula, isFormula: true }]
|
||||
);
|
||||
private async _setColumnFormula(transformColRef: number, formula: string|null, info: SourceInfo) {
|
||||
if (formula === null) {
|
||||
await this._gristDoc.clearColumns([transformColRef], {keepType: true});
|
||||
} else {
|
||||
await this._gristDoc.docModel.columns.sendTableAction(
|
||||
['UpdateRecord', transformColRef, { formula, isFormula: true }]);
|
||||
}
|
||||
await this._updateImportDiff(info);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -841,26 +860,20 @@ export class Importer extends DisposableWithEvents {
|
||||
* in the column mapping section of Importer. On click, opens
|
||||
* an editor for the formula for `column`.
|
||||
*/
|
||||
private _buildColMappingFormula(_owner: MultiHolder, column: ColumnRec, buildEditor: (e: Element) => void,
|
||||
placeholder: string) {
|
||||
const formatFormula = (formula: string) => {
|
||||
const sourceColLabels = this._sourceColLabelsById.get();
|
||||
if (!sourceColLabels) { return formula; }
|
||||
private _buildColMappingFormula(owner: MultiHolder, field: ViewFieldRec, info: SourceInfo) {
|
||||
const displayFormula = Computed.create(owner, use => {
|
||||
const column = use(field.column);
|
||||
const formula = use(column.formula);
|
||||
const importOptions = use(this._transformColImportOptions).get(column.getRowId());
|
||||
return importOptions?.get(formula) ?? formula;
|
||||
});
|
||||
|
||||
formula = formula.trim();
|
||||
if (formula.startsWith('$') && sourceColLabels.has(formula.slice(1))) {
|
||||
// For simple formulas that only reference a source column id, show the source column label.
|
||||
return sourceColLabels.get(formula.slice(1))!;
|
||||
}
|
||||
|
||||
return formula;
|
||||
};
|
||||
|
||||
return cssFieldFormula(use => formatFormula(use(column.formula)),
|
||||
{gristTheme: this._gristDoc.currentTheme, placeholder, maxLines: 1},
|
||||
return cssFieldFormula(displayFormula,
|
||||
{gristTheme: this._gristDoc.currentTheme, placeholder: 'Skip', maxLines: 1},
|
||||
dom.cls('disabled'),
|
||||
{tabIndex: '-1'},
|
||||
dom.on('focus', (_ev, elem) => buildEditor(elem)),
|
||||
dom.on('focus', (_ev, elem) =>
|
||||
this._activateFormulaEditor(elem, field, () => this._updateImportDiff(info))),
|
||||
testId('importer-column-match-formula'),
|
||||
);
|
||||
}
|
||||
|
||||
@ -70,7 +70,7 @@ export type IsRightTypeFunc = (value: CellValue) => boolean;
|
||||
export class BaseFormatter {
|
||||
protected isRightType: IsRightTypeFunc;
|
||||
|
||||
constructor(public type: string, public widgetOpts: object, public docSettings: DocumentSettings) {
|
||||
constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) {
|
||||
this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) ||
|
||||
gristTypes.isRightType('Any')!;
|
||||
}
|
||||
@ -144,7 +144,7 @@ export interface DateFormatOptions {
|
||||
}
|
||||
|
||||
class DateFormatter extends BaseFormatter {
|
||||
private _dateTimeFormat: string;
|
||||
protected _dateTimeFormat: string;
|
||||
private _timezone: string;
|
||||
|
||||
constructor(type: string, widgetOpts: DateFormatOptions, docSettings: DocumentSettings, timezone: string = 'UTC') {
|
||||
@ -194,9 +194,11 @@ export interface DateTimeFormatOptions extends DateFormatOptions {
|
||||
class DateTimeFormatter extends DateFormatter {
|
||||
constructor(type: string, widgetOpts: DateTimeFormatOptions, docSettings: DocumentSettings) {
|
||||
const timezone = gutil.removePrefix(type, "DateTime:") || '';
|
||||
// Pass up the original widgetOpts. It's helpful to have them available; e.g. ExcelFormatter
|
||||
// takes options from an initialized ValueFormatter.
|
||||
super(type, widgetOpts, docSettings, timezone);
|
||||
const timeFormat = widgetOpts.timeFormat === undefined ? 'h:mma' : widgetOpts.timeFormat;
|
||||
const dateFormat = (widgetOpts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
|
||||
super(type, {dateFormat}, docSettings, timezone);
|
||||
this._dateTimeFormat = (widgetOpts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -268,7 +268,8 @@ export function createParserOrFormatterArguments(
|
||||
const col = columnsTable.getRecord(colRef)!;
|
||||
let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col;
|
||||
if (fieldRef) {
|
||||
fieldOrCol = fieldsTable.getRecord(fieldRef) || col;
|
||||
const field = fieldsTable.getRecord(fieldRef);
|
||||
fieldOrCol = field?.widgetOptions ? field : col;
|
||||
}
|
||||
|
||||
return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol);
|
||||
|
||||
@ -5,11 +5,11 @@ import * as _ from 'underscore';
|
||||
|
||||
import {ColumnDelta, createEmptyActionSummary} from 'app/common/ActionSummary';
|
||||
import {ApplyUAResult, DataSourceTransformed, ImportOptions, ImportResult, ImportTableResult,
|
||||
MergeOptions, MergeOptionsMap, MergeStrategy, SKIP_TABLE, TransformColumn,
|
||||
MergeOptions, MergeOptionsMap, MergeStrategy, SKIP_TABLE,
|
||||
TransformRule,
|
||||
TransformRuleMap} from 'app/common/ActiveDocAPI';
|
||||
import {ApiError} from 'app/common/ApiError';
|
||||
import {BulkColValues, CellValue, fromTableDataAction, TableRecordValue, UserAction} from 'app/common/DocActions';
|
||||
import {BulkColValues, CellValue, fromTableDataAction, UserAction} from 'app/common/DocActions';
|
||||
import * as gutil from 'app/common/gutil';
|
||||
import {DocStateComparison} from 'app/common/UserAPI';
|
||||
import {guessColInfoForImports} from 'app/common/ValueGuesser';
|
||||
@ -339,9 +339,9 @@ export class ActiveDocImport {
|
||||
if (isHidden) {
|
||||
// Generate formula columns, view sections, etc
|
||||
const results: ApplyUAResult = await this._activeDoc.applyUserActions(docSession,
|
||||
[['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null]]);
|
||||
[['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null, null]]);
|
||||
|
||||
transformSectionRef = results.retValues[0];
|
||||
transformSectionRef = results.retValues[0].viewSectionRef;
|
||||
createdTableId = hiddenTableId;
|
||||
|
||||
} else {
|
||||
@ -391,36 +391,21 @@ export class ActiveDocImport {
|
||||
* the source and destination table.
|
||||
* @returns {string} The table id of the new or updated destination table.
|
||||
*/
|
||||
private async _transformAndFinishImport(docSession: OptDocSession,
|
||||
hiddenTableId: string, destTableId: string,
|
||||
intoNewTable: boolean, transformRule: TransformRule|null,
|
||||
mergeOptions: MergeOptions|null): Promise<string> {
|
||||
private async _transformAndFinishImport(
|
||||
docSession: OptDocSession,
|
||||
hiddenTableId: string, destTableId: string,
|
||||
intoNewTable: boolean, transformRule: TransformRule|null,
|
||||
mergeOptions: MergeOptions|null
|
||||
): Promise<string> {
|
||||
log.info("ActiveDocImport._transformAndFinishImport(%s, %s, %s, %s, %s)",
|
||||
hiddenTableId, destTableId, intoNewTable, transformRule, mergeOptions);
|
||||
const srcCols = await this._activeDoc.getTableCols(docSession, hiddenTableId);
|
||||
|
||||
// Use a default transform rule if one was not provided by the client.
|
||||
if (!transformRule) {
|
||||
const transformDest = intoNewTable ? null : destTableId;
|
||||
transformRule = await this._makeDefaultTransformRule(docSession, srcCols, transformDest);
|
||||
}
|
||||
|
||||
// Transform rules from client may have prefixed column ids, so we need to strip them.
|
||||
stripRulePrefixes(transformRule);
|
||||
|
||||
if (intoNewTable) {
|
||||
// Transform rules for new tables don't have filled in destination column ids.
|
||||
const result = await this._activeDoc.applyUserActions(docSession, [['FillTransformRuleColIds', transformRule]]);
|
||||
transformRule = result.retValues[0] as TransformRule;
|
||||
|
||||
// Encode Refs as Ints, to avoid table dependency issues. We'll convert back to Ref at the end.
|
||||
encodeRuleReferences(transformRule);
|
||||
} else if (transformRule.destCols.some(c => c.colId === null)) {
|
||||
throw new Error('Column ids in transform rule must be filled when importing into an existing table');
|
||||
}
|
||||
|
||||
await this._activeDoc.applyUserActions(docSession,
|
||||
[['MakeImportTransformColumns', hiddenTableId, transformRule, false]]);
|
||||
const transformDestTableId = intoNewTable ? null : destTableId;
|
||||
const result = await this._activeDoc.applyUserActions(docSession, [[
|
||||
'GenImporterView', hiddenTableId, transformDestTableId, transformRule,
|
||||
{createViewSection: false, genAll: false, refsAsInts: true},
|
||||
]]);
|
||||
transformRule = result.retValues[0].transformRule as TransformRule;
|
||||
|
||||
if (!intoNewTable && mergeOptions && mergeOptions.mergeCols.length > 0) {
|
||||
await this._mergeAndFinishImport(docSession, hiddenTableId, destTableId, transformRule, mergeOptions);
|
||||
@ -430,6 +415,7 @@ export class ActiveDocImport {
|
||||
const hiddenTableData = fromTableDataAction(await this._activeDoc.fetchTable(docSession, hiddenTableId, true));
|
||||
const columnData: BulkColValues = {};
|
||||
|
||||
const srcCols = await this._activeDoc.getTableCols(docSession, hiddenTableId);
|
||||
const srcColIds = srcCols.map(c => c.id as string);
|
||||
|
||||
// Only include destination columns that weren't skipped.
|
||||
@ -591,38 +577,6 @@ export class ActiveDocImport {
|
||||
return this._activeDoc.docStorage.decodeMarshalledDataFromTables(result);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a default TransformRule using column definitions from `destTableId`. If `destTableId`
|
||||
* is null (in the case when the import destination is a new table), the `srcCols` are used instead.
|
||||
*
|
||||
* @param {TableRecordValue[]} srcCols Source column definitions.
|
||||
* @param {string|null} destTableId The destination table id. If null, the destination is assumed
|
||||
* to be a new table, and `srcCols` are used to build the transform rule.
|
||||
* @returns {Promise<TransformRule>} The constructed transform rule.
|
||||
*/
|
||||
private async _makeDefaultTransformRule(docSession: OptDocSession, srcCols: TableRecordValue[],
|
||||
destTableId: string|null): Promise<TransformRule> {
|
||||
const targetCols = destTableId ? await this._activeDoc.getTableCols(docSession, destTableId) : srcCols;
|
||||
const destCols: TransformColumn[] = [];
|
||||
const srcColIds = srcCols.map(c => c.id as string);
|
||||
|
||||
for (const {id, fields} of targetCols) {
|
||||
destCols.push({
|
||||
colId: destTableId ? id as string : null,
|
||||
label: fields.label as string,
|
||||
type: fields.type as string,
|
||||
widgetOptions: fields.widgetOptions as string,
|
||||
formula: srcColIds.includes(id as string) ? `$${id}` : ''
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
destTableId,
|
||||
destCols,
|
||||
sourceCols: srcColIds
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* This function removes temporary hidden tables which were created during the import process
|
||||
*
|
||||
@ -694,32 +648,6 @@ function isBlank(value: CellValue): boolean {
|
||||
return value === null || (typeof value === 'string' && value.trim().length === 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Changes every Ref column to an Int column in `destCols`.
|
||||
*
|
||||
* Encoding references as ints can be useful when finishing imports to avoid
|
||||
* issues such as importing linked tables in the wrong order. When encoding references,
|
||||
* ActiveDocImport._fixReferences should be called at the end of importing to
|
||||
* decode Ints back to Refs.
|
||||
*/
|
||||
function encodeRuleReferences({destCols}: TransformRule): void {
|
||||
for (const col of destCols) {
|
||||
const refTableId = gutil.removePrefix(col.type, "Ref:");
|
||||
if (refTableId) {
|
||||
col.type = 'Int';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function that strips import prefixes from columns in transform rules (if ids are present).
|
||||
function stripRulePrefixes({destCols}: TransformRule): void {
|
||||
for (const col of destCols) {
|
||||
const colId = col.colId;
|
||||
if (colId && colId.startsWith(IMPORT_TRANSFORM_COLUMN_PREFIX)) {
|
||||
col.colId = colId.slice(IMPORT_TRANSFORM_COLUMN_PREFIX.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function that returns new `colIds` with import prefixes stripped.
|
||||
function stripPrefixes(colIds: string[]): string[] {
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import {ApiError} from 'app/common/ApiError';
|
||||
import {buildColFilter} from 'app/common/ColumnFilterFunc';
|
||||
import {TableDataAction} from 'app/common/DocActions';
|
||||
import {DocData} from 'app/common/DocData';
|
||||
import {DocumentSettings} from 'app/common/DocumentSettings';
|
||||
import * as gristTypes from 'app/common/gristTypes';
|
||||
import * as gutil from 'app/common/gutil';
|
||||
@ -11,6 +12,7 @@ import {schema, SchemaTypes} from 'app/common/schema';
|
||||
import {SortFunc} from 'app/common/SortFunc';
|
||||
import {Sort} from 'app/common/SortSpec';
|
||||
import {MetaRowRecord, MetaTableData} from 'app/common/TableData';
|
||||
import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFormatter';
|
||||
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
|
||||
import {RequestWithLogin} from 'app/server/lib/Authorizer';
|
||||
import {docSessionFromRequest} from 'app/server/lib/DocSession';
|
||||
@ -23,23 +25,16 @@ import * as _ from 'underscore';
|
||||
type Access = (row: number) => any;
|
||||
|
||||
// Helper interface with information about the column
|
||||
interface ExportColumn {
|
||||
export interface ExportColumn {
|
||||
id: number;
|
||||
colId: string;
|
||||
label: string;
|
||||
type: string;
|
||||
widgetOptions: any;
|
||||
formatter: BaseFormatter;
|
||||
parentPos: number;
|
||||
description: string;
|
||||
}
|
||||
// helper for empty column
|
||||
const emptyCol: ExportColumn = {
|
||||
id: 0,
|
||||
colId: '',
|
||||
label: '',
|
||||
type: '',
|
||||
widgetOptions: null,
|
||||
parentPos: 0
|
||||
};
|
||||
|
||||
/**
|
||||
* Bare data that is exported - used to convert to various formats.
|
||||
*/
|
||||
@ -173,41 +168,33 @@ export async function exportTable(
|
||||
{metaTables}: {metaTables?: FilteredMetaTables} = {},
|
||||
): Promise<ExportData> {
|
||||
metaTables = metaTables || await getMetaTables(activeDoc, req);
|
||||
const docData = new DocData((tableId) => { throw new Error("Unexpected DocData fetch"); }, metaTables);
|
||||
const tables = safeTable(metaTables, '_grist_Tables');
|
||||
const metaColumns = safeTable(metaTables, '_grist_Tables_column');
|
||||
checkTableAccess(tables, tableRef);
|
||||
const table = safeRecord(tables, tableRef);
|
||||
const tableColumns = safeTable(metaTables, '_grist_Tables_column')
|
||||
.getRecords()
|
||||
|
||||
// Select only columns that belong to this table.
|
||||
const tableColumns = metaColumns.filterRecords({parentId: tableRef})
|
||||
// sort by parentPos and id, which should be the same order as in raw data
|
||||
.sort((c1, c2) => nativeCompare(c1.parentPos, c2.parentPos) || nativeCompare(c1.id, c2.id))
|
||||
// remove manual sort column
|
||||
.filter(col => col.colId !== gristTypes.MANUALSORT);
|
||||
.sort((c1, c2) => nativeCompare(c1.parentPos, c2.parentPos) || nativeCompare(c1.id, c2.id));
|
||||
|
||||
// Produce a column description matching what user will see / expect to export
|
||||
const tableColsById = _.indexBy(tableColumns, 'id');
|
||||
const columns = tableColumns.map(tc => {
|
||||
// remove all columns that don't belong to this table
|
||||
if (tc.parentId !== tableRef) {
|
||||
return emptyCol;
|
||||
}
|
||||
// remove all helpers
|
||||
if (gristTypes.isHiddenCol(tc.colId)) {
|
||||
return emptyCol;
|
||||
}
|
||||
const columns: ExportColumn[] = tableColumns
|
||||
.filter(tc => !gristTypes.isHiddenCol(tc.colId)) // Exclude helpers
|
||||
.map<ExportColumn>(tc => {
|
||||
// for reference columns, return display column, and copy settings from visible column
|
||||
const displayCol = tableColsById[tc.displayCol || tc.id];
|
||||
const colOptions = gutil.safeJsonParse(tc.widgetOptions, {});
|
||||
const displayOptions = gutil.safeJsonParse(displayCol.widgetOptions, {});
|
||||
const widgetOptions = Object.assign(displayOptions, colOptions);
|
||||
const displayCol = metaColumns.getRecord(tc.displayCol) || tc;
|
||||
return {
|
||||
id: displayCol.id,
|
||||
colId: displayCol.colId,
|
||||
label: tc.label,
|
||||
type: displayCol.type,
|
||||
widgetOptions,
|
||||
type: tc.type,
|
||||
formatter: createFullFormatterFromDocData(docData, tc.id),
|
||||
parentPos: tc.parentPos,
|
||||
description: displayCol.description,
|
||||
description: tc.description,
|
||||
};
|
||||
}).filter(tc => tc !== emptyCol);
|
||||
});
|
||||
|
||||
// fetch actual data
|
||||
const {tableData} = await activeDoc.fetchTable(docSessionFromRequest(req as RequestWithLogin), table.tableId, true);
|
||||
@ -251,37 +238,35 @@ export async function exportSection(
|
||||
{metaTables}: {metaTables?: FilteredMetaTables} = {},
|
||||
): Promise<ExportData> {
|
||||
metaTables = metaTables || await getMetaTables(activeDoc, req);
|
||||
const docData = new DocData((tableId) => { throw new Error("Unexpected DocData fetch"); }, metaTables);
|
||||
const viewSections = safeTable(metaTables, '_grist_Views_section');
|
||||
const viewSection = safeRecord(viewSections, viewSectionId);
|
||||
safe(viewSection.tableRef, `Cannot find or access table`);
|
||||
const tables = safeTable(metaTables, '_grist_Tables');
|
||||
checkTableAccess(tables, viewSection.tableRef);
|
||||
const table = safeRecord(tables, viewSection.tableRef);
|
||||
const columns = safeTable(metaTables, '_grist_Tables_column')
|
||||
.filterRecords({parentId: table.id});
|
||||
const metaColumns = safeTable(metaTables, '_grist_Tables_column');
|
||||
const columns = metaColumns.filterRecords({parentId: table.id});
|
||||
const viewSectionFields = safeTable(metaTables, '_grist_Views_section_field');
|
||||
const fields = viewSectionFields.filterRecords({parentId: viewSection.id});
|
||||
const savedFilters = safeTable(metaTables, '_grist_Filters')
|
||||
.filterRecords({viewSectionRef: viewSection.id});
|
||||
|
||||
const tableColsById = _.indexBy(columns, 'id');
|
||||
const fieldsByColRef = _.indexBy(fields, 'colRef');
|
||||
const savedFiltersByColRef = _.indexBy(savedFilters, 'colRef');
|
||||
const unsavedFiltersByColRef = _.indexBy(filters ?? [], 'colRef');
|
||||
|
||||
// Produce a column description matching what user will see / expect to export
|
||||
const viewify = (col: GristTablesColumn, field?: GristViewsSectionField) => {
|
||||
const displayCol = tableColsById[field?.displayCol || col.displayCol || col.id];
|
||||
const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {});
|
||||
const fieldWidgetOptions = field ? gutil.safeJsonParse(field.widgetOptions, {}) : {};
|
||||
const viewify = (col: GristTablesColumn, field?: GristViewsSectionField): ExportColumn => {
|
||||
const displayCol = metaColumns.getRecord(field?.displayCol || col.displayCol) || col;
|
||||
return {
|
||||
id: displayCol.id,
|
||||
colId: displayCol.colId,
|
||||
label: col.label,
|
||||
type: col.type,
|
||||
formatter: createFullFormatterFromDocData(docData, col.id, field?.id),
|
||||
parentPos: col.parentPos,
|
||||
description: col.description,
|
||||
widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions),
|
||||
};
|
||||
};
|
||||
const buildFilters = (col: GristTablesColumn, field?: GristViewsSectionField) => {
|
||||
@ -297,14 +282,14 @@ export async function exportSection(
|
||||
const columnsForFilters = columns
|
||||
.filter(column => !gristTypes.isHiddenCol(column.colId))
|
||||
.map(column => buildFilters(column, fieldsByColRef[column.id]));
|
||||
const viewColumns = _.sortBy(fields, 'parentPos')
|
||||
.map((field) => viewify(tableColsById[field.colRef], field));
|
||||
const viewColumns: ExportColumn[] = _.sortBy(fields, 'parentPos')
|
||||
.map((field) => viewify(metaColumns.getRecord(field.colRef)!, field));
|
||||
|
||||
// The columns named in sort order need to now become display columns
|
||||
sortSpec = sortSpec || gutil.safeJsonParse(viewSection.sortColRefs, []);
|
||||
sortSpec = sortSpec!.map((colSpec) => {
|
||||
const colRef = Sort.getColRef(colSpec);
|
||||
const col = tableColsById[colRef];
|
||||
const col = metaColumns.getRecord(colRef);
|
||||
if (!col) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
import {ApiError} from 'app/common/ApiError';
|
||||
import {createFormatter} from 'app/common/ValueFormatter';
|
||||
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
|
||||
import {DownloadOptions, ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export';
|
||||
import log from 'app/server/lib/log';
|
||||
@ -86,7 +85,7 @@ function convertToCsv({
|
||||
}: ExportData) {
|
||||
|
||||
// create formatters for columns
|
||||
const formatters = viewColumns.map(col => createFormatter(col.type, col.widgetOptions, docSettings));
|
||||
const formatters = viewColumns.map(col => col.formatter);
|
||||
// Arrange the data into a row-indexed matrix, starting with column headers.
|
||||
const csvMatrix = [viewColumns.map(col => col.label)];
|
||||
// populate all the rows with values as strings
|
||||
|
||||
@ -1,18 +1,7 @@
|
||||
import * as express from 'express';
|
||||
import {ApiError} from 'app/common/ApiError';
|
||||
import {WidgetOptions} from 'app/common/WidgetOptions';
|
||||
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
|
||||
import {DownloadOptions, exportTable} from 'app/server/lib/Export';
|
||||
|
||||
interface ExportColumn {
|
||||
id: number;
|
||||
colId: string;
|
||||
label: string;
|
||||
type: string;
|
||||
widgetOptions: WidgetOptions;
|
||||
description?: string;
|
||||
parentPos: number;
|
||||
}
|
||||
import {DownloadOptions, ExportColumn, exportTable} from 'app/server/lib/Export';
|
||||
|
||||
interface FrictionlessFormat {
|
||||
name: string;
|
||||
@ -86,35 +75,36 @@ function columnsToTableSchema(
|
||||
|
||||
function buildTypeField(col: ExportColumn, locale: string) {
|
||||
const type = col.type.split(':', 1)[0];
|
||||
const widgetOptions = col.formatter.widgetOpts;
|
||||
switch (type) {
|
||||
case 'Text':
|
||||
return {
|
||||
type: 'string',
|
||||
format: col.widgetOptions.widget === 'HyperLink' ? 'uri' : 'default',
|
||||
format: widgetOptions.widget === 'HyperLink' ? 'uri' : 'default',
|
||||
};
|
||||
case 'Numeric':
|
||||
return {
|
||||
type: 'number',
|
||||
bareNumber: col.widgetOptions?.numMode === 'decimal',
|
||||
bareNumber: widgetOptions?.numMode === 'decimal',
|
||||
...getNumberSeparators(locale),
|
||||
};
|
||||
case 'Integer':
|
||||
return {
|
||||
type: 'integer',
|
||||
bareNumber: col.widgetOptions?.numMode === 'decimal',
|
||||
bareNumber: widgetOptions?.numMode === 'decimal',
|
||||
groupChar: getNumberSeparators(locale).groupChar,
|
||||
};
|
||||
case 'Date':
|
||||
return {
|
||||
type: 'date',
|
||||
format: 'any',
|
||||
gristFormat: col.widgetOptions?.dateFormat || 'YYYY-MM-DD',
|
||||
gristFormat: widgetOptions?.dateFormat || 'YYYY-MM-DD',
|
||||
};
|
||||
case 'DateTime':
|
||||
return {
|
||||
type: 'datetime',
|
||||
format: 'any',
|
||||
gristFormat: `${col.widgetOptions?.dateFormat} ${col.widgetOptions?.timeFormat}`,
|
||||
gristFormat: `${widgetOptions?.dateFormat} ${widgetOptions?.timeFormat}`,
|
||||
};
|
||||
case 'Bool':
|
||||
return {
|
||||
@ -125,12 +115,12 @@ function buildTypeField(col: ExportColumn, locale: string) {
|
||||
case 'Choice':
|
||||
return {
|
||||
type: 'string',
|
||||
constraints: {enum: col.widgetOptions?.choices},
|
||||
constraints: {enum: widgetOptions?.choices},
|
||||
};
|
||||
case 'ChoiceList':
|
||||
return {
|
||||
type: 'array',
|
||||
constraints: {enum: col.widgetOptions?.choices},
|
||||
constraints: {enum: widgetOptions?.choices},
|
||||
};
|
||||
case 'Reference':
|
||||
return {type: 'string'};
|
||||
|
||||
@ -130,7 +130,7 @@ async function convertToExcel(tables: ExportData[], testDates: boolean) {
|
||||
const { columns, rowIds, access, tableName } = table;
|
||||
const ws = wb.addWorksheet(sanitizeWorksheetName(tableName));
|
||||
// Build excel formatters.
|
||||
const formatters = columns.map(col => createExcelFormatter(col.type, col.widgetOptions));
|
||||
const formatters = columns.map(col => createExcelFormatter(col.formatter.type, col.formatter.widgetOpts));
|
||||
// Generate headers for all columns with correct styles for whole column.
|
||||
// Actual header style for a first row will be overwritten later.
|
||||
ws.columns = columns.map((col, c) => ({ header: col.label, style: formatters[c].style() }));
|
||||
|
||||
@ -5,10 +5,9 @@ from __future__ import absolute_import
|
||||
import datetime
|
||||
import math as _math
|
||||
import operator
|
||||
import os
|
||||
import random
|
||||
import uuid
|
||||
from functools import reduce
|
||||
from functools import reduce # pylint: disable=redefined-builtin
|
||||
|
||||
from six.moves import zip, xrange
|
||||
import six
|
||||
@ -491,6 +490,25 @@ def MULTINOMIAL(value1, *more_values):
|
||||
res *= COMBIN(s, v)
|
||||
return res
|
||||
|
||||
def NUM(value):
|
||||
"""
|
||||
For a Python floating-point value that's actually an integer, returns a Python integer type.
|
||||
Otherwise, returns the value unchanged. This is helpful sometimes when a value comes from a
|
||||
Numeric Grist column (represented as floats), but when int values are actually expected.
|
||||
|
||||
>>> NUM(-17.0)
|
||||
-17
|
||||
>>> NUM(1.5)
|
||||
1.5
|
||||
>>> NUM(4)
|
||||
4
|
||||
>>> NUM("NA")
|
||||
'NA'
|
||||
"""
|
||||
if isinstance(value, float) and value.is_integer():
|
||||
return int(value)
|
||||
return value
|
||||
|
||||
def ODD(value):
|
||||
"""
|
||||
Rounds a number up to the nearest odd integer.
|
||||
@ -869,10 +887,12 @@ def TRUNC(value, places=0):
|
||||
def UUID():
|
||||
"""
|
||||
Generate a random UUID-formatted string identifier.
|
||||
|
||||
Since UUID() produces a different value each time it's called, it is best to use it in
|
||||
[trigger formula](formulas.md#trigger-formulas) for new records.
|
||||
This would only calculate UUID() once and freeze the calculated value. By contrast, a regular formula
|
||||
may get recalculated any time the document is reloaded, producing a different value for UUID() each time.
|
||||
This would only calculate UUID() once and freeze the calculated value. By contrast, a regular
|
||||
formula may get recalculated any time the document is reloaded, producing a different value for
|
||||
UUID() each time.
|
||||
"""
|
||||
try:
|
||||
uid = uuid.uuid4()
|
||||
|
||||
@ -1,5 +1,3 @@
|
||||
from collections import namedtuple
|
||||
|
||||
from six.moves import zip
|
||||
|
||||
import column
|
||||
@ -86,7 +84,7 @@ class ImportActions(object):
|
||||
hidden_table_rec = tables.lookupOne(tableId=hidden_table_id)
|
||||
|
||||
# will use these to set default formulas (if column names match in src and dest table)
|
||||
src_cols = {c.colId for c in hidden_table_rec.columns}
|
||||
src_cols = {c.colId: c for c in hidden_table_rec.columns}
|
||||
|
||||
target_table = tables.lookupOne(tableId=dest_table_id) if dest_table_id else hidden_table_rec
|
||||
target_cols = target_table.columns
|
||||
@ -97,34 +95,29 @@ class ImportActions(object):
|
||||
dest_cols = []
|
||||
for c in target_cols:
|
||||
if column.is_visible_column(c.colId) and (not c.isFormula or c.formula == ""):
|
||||
|
||||
dest_cols.append( {
|
||||
source_col = src_cols.get(c.colId)
|
||||
dest_col = {
|
||||
"label": c.label,
|
||||
"colId": c.colId if dest_table_id else None, #should be None if into new table
|
||||
"type": c.type,
|
||||
"widgetOptions": getattr(c, "widgetOptions", ""),
|
||||
"formula": ("$" + c.colId) if (c.colId in src_cols) else ''
|
||||
})
|
||||
"formula": ("$" + source_col.colId) if source_col else '',
|
||||
}
|
||||
if source_col and c.type.startswith("Ref:"):
|
||||
ref_table_id = c.type.split(':')[1]
|
||||
visible_col = c.visibleCol
|
||||
if visible_col:
|
||||
dest_col["visibleCol"] = visible_col.id
|
||||
dest_col["formula"] = '{}.lookupOne({}=${c}) or (${c} and str(${c}))'.format(
|
||||
ref_table_id, visible_col.colId, c=source_col.colId)
|
||||
|
||||
dest_cols.append(dest_col)
|
||||
|
||||
return {"destCols": dest_cols}
|
||||
# doesnt generate other fields of transform_rule, but sandbox only used destCols
|
||||
|
||||
|
||||
def FillTransformRuleColIds(self, transform_rule):
|
||||
"""
|
||||
Takes a transform rule with missing dest col ids, and returns it
|
||||
with sanitized and de-duplicated ids generated from the original
|
||||
column labels.
|
||||
|
||||
NOTE: This work could be done outside the data engine, but the logic
|
||||
for cleaning column identifiers is quite complex and currently only lives
|
||||
in the data engine. In the future, it may be worth porting it to
|
||||
Node to avoid an extra trip to the data engine.
|
||||
"""
|
||||
return _gen_colids(transform_rule)
|
||||
|
||||
|
||||
def MakeImportTransformColumns(self, hidden_table_id, transform_rule, gen_all):
|
||||
def _MakeImportTransformColumns(self, hidden_table_id, transform_rule, gen_all):
|
||||
"""
|
||||
Makes prefixed columns in the grist hidden import table (hidden_table_id)
|
||||
|
||||
@ -142,10 +135,9 @@ class ImportActions(object):
|
||||
src_cols = {c.colId for c in hidden_table_rec.columns}
|
||||
log.debug("destCols:" + repr(transform_rule['destCols']))
|
||||
|
||||
#wrap dest_cols as namedtuples, to allow access like 'dest_col.param'
|
||||
dest_cols = [namedtuple('col', c.keys())(*c.values()) for c in transform_rule['destCols']]
|
||||
dest_cols = transform_rule['destCols']
|
||||
|
||||
log.debug("MakeImportTransformColumns: {}".format("gen_all" if gen_all else "optimize"))
|
||||
log.debug("_MakeImportTransformColumns: {}".format("gen_all" if gen_all else "optimize"))
|
||||
|
||||
# Calling rebuild_usercode once per added column is wasteful and can be very slow.
|
||||
self._engine._should_rebuild_usercode = False
|
||||
@ -156,21 +148,31 @@ class ImportActions(object):
|
||||
try:
|
||||
for c in dest_cols:
|
||||
# skip copy columns (unless gen_all)
|
||||
formula = c.formula.strip()
|
||||
formula = c["formula"].strip()
|
||||
isCopyFormula = (formula.startswith("$") and formula[1:] in src_cols)
|
||||
|
||||
if gen_all or not isCopyFormula:
|
||||
# If colId specified, use that. Otherwise, use the (sanitized) label.
|
||||
col_id = c.colId or identifiers.pick_col_ident(c.label)
|
||||
col_id = c["colId"] or identifiers.pick_col_ident(c["label"])
|
||||
visible_col_ref = c.get("visibleCol", 0)
|
||||
new_col_id = _import_transform_col_prefix + col_id
|
||||
new_col_spec = {
|
||||
"label": c.label,
|
||||
"type": c.type,
|
||||
"widgetOptions": getattr(c, "widgetOptions", ""),
|
||||
"label": c["label"],
|
||||
"type": c["type"],
|
||||
"widgetOptions": c.get("widgetOptions", ""),
|
||||
"isFormula": True,
|
||||
"formula": c.formula}
|
||||
"formula": c["formula"],
|
||||
"visibleCol": visible_col_ref,
|
||||
}
|
||||
result = self._useractions.doAddColumn(hidden_table_id, new_col_id, new_col_spec)
|
||||
new_cols.append(result["colRef"])
|
||||
new_col_id, new_col_ref = result["colId"], result["colRef"]
|
||||
|
||||
if visible_col_ref:
|
||||
visible_col_id = self._docmodel.columns.table.get_record(visible_col_ref).colId
|
||||
self._useractions.SetDisplayFormula(hidden_table_id, None, new_col_ref,
|
||||
'${}.{}'.format(new_col_id, visible_col_id))
|
||||
|
||||
new_cols.append(new_col_ref)
|
||||
finally:
|
||||
self._engine._should_rebuild_usercode = True
|
||||
self._engine.rebuild_usercode()
|
||||
@ -178,36 +180,40 @@ class ImportActions(object):
|
||||
return new_cols
|
||||
|
||||
|
||||
def DoGenImporterView(self, source_table_id, dest_table_id, transform_rule = None):
|
||||
def DoGenImporterView(self, source_table_id, dest_table_id, transform_rule, options):
|
||||
"""
|
||||
Generates viewsections/formula columns for importer
|
||||
Generates formula columns for transformed importer columns, and optionally a new viewsection.
|
||||
|
||||
source_table_id: id of temporary hidden table, data parsed from data source
|
||||
dest_table_id: id of table to import to, or None for new table
|
||||
source_table_id: id of temporary hidden table, containing source data and used for preview.
|
||||
dest_table_id: id of table to import to, or None for new table.
|
||||
transform_rule: transform_rule to reuse (if it still applies), if None will generate new one
|
||||
options: a dictionary with optional keys:
|
||||
createViewSection: defaults to True, in which case creates a new view-section to show the
|
||||
generated columns, for use in review, and remove any previous ones.
|
||||
genAll: defaults to True; if False, transform formulas that just copy will not be generated.
|
||||
refsAsInts: if set, treat Ref columns as type Int for a new dest_table. This is used when
|
||||
finishing imports from multi-table sources (e.g. from json) to avoid issues such as
|
||||
importing linked tables in the wrong order. Caller is expected to fix these up separately.
|
||||
|
||||
Removes old transform viewSection and columns for source_table_id, and creates new ones that
|
||||
match the destination table.
|
||||
Returns the rowId of the newly added section or 0 if no source table (source_table_id
|
||||
can be None in case of importing empty file).
|
||||
|
||||
Creates formula columns for transforms (match columns in dest table)
|
||||
Returns and object with:
|
||||
transformRule: updated (normalized) transform rule, or a newly generated one.
|
||||
viewSectionRef: rowId of the newly added section, present only if createViewSection is set.
|
||||
"""
|
||||
createViewSection = options.get("createViewSection", True)
|
||||
genAll = options.get("genAll", True)
|
||||
refsAsInts = options.get("refsAsInts", True)
|
||||
|
||||
tables = self._docmodel.tables
|
||||
src_table_rec = tables.lookupOne(tableId=source_table_id)
|
||||
if createViewSection:
|
||||
src_table_rec = self._docmodel.tables.lookupOne(tableId=source_table_id)
|
||||
|
||||
# for new table, dest_table_id is None
|
||||
dst_table_rec = tables.lookupOne(tableId=dest_table_id) if dest_table_id else src_table_rec
|
||||
# ======== Cleanup old sections/columns
|
||||
|
||||
# ======== Cleanup old sections/columns
|
||||
|
||||
# Transform columns are those that start with a special prefix.
|
||||
old_cols = [c for c in src_table_rec.columns
|
||||
if c.colId.startswith(_import_transform_col_prefix)]
|
||||
old_sections = {field.parentId for c in old_cols for field in c.viewFields}
|
||||
self._docmodel.remove(old_sections)
|
||||
self._docmodel.remove(old_cols)
|
||||
# Transform columns are those that start with a special prefix.
|
||||
old_cols = [c for c in src_table_rec.columns
|
||||
if c.colId.startswith(_import_transform_col_prefix)]
|
||||
old_sections = {field.parentId for c in old_cols for field in c.viewFields}
|
||||
self._docmodel.remove(old_sections)
|
||||
self._docmodel.remove(old_cols)
|
||||
|
||||
#======== Prepare/normalize transform_rule, Create new formula columns
|
||||
# Defaults to duplicating dest_table columns (or src_table columns for a new table)
|
||||
@ -216,26 +222,35 @@ class ImportActions(object):
|
||||
if transform_rule is None:
|
||||
transform_rule = self._MakeDefaultTransformRule(source_table_id, dest_table_id)
|
||||
|
||||
else: #ensure prefixes, colIds are correct
|
||||
_strip_prefixes(transform_rule)
|
||||
# ensure prefixes, colIds are correct
|
||||
_strip_prefixes(transform_rule)
|
||||
|
||||
if not dest_table_id: # into new table: 'colId's are undefined
|
||||
_gen_colids(transform_rule)
|
||||
else:
|
||||
if None in (dc["colId"] for dc in transform_rule["destCols"]):
|
||||
errstr = "colIds must be defined in transform_rule for importing into existing table: "
|
||||
raise ValueError(errstr + repr(transform_rule))
|
||||
if not dest_table_id: # into new table: 'colId's are undefined
|
||||
_gen_colids(transform_rule)
|
||||
|
||||
# Treat destination Ref:* columns as Int instead, for new tables, to avoid issues when
|
||||
# importing linked tables in the wrong order. Caller is expected to fix up afterwards.
|
||||
if refsAsInts:
|
||||
for col in transform_rule["destCols"]:
|
||||
if col["type"].startswith("Ref:"):
|
||||
col["type"] = "Int"
|
||||
|
||||
new_cols = self.MakeImportTransformColumns(source_table_id, transform_rule, gen_all=True)
|
||||
# we want to generate all columns so user can see them and edit
|
||||
else:
|
||||
if None in (dc["colId"] for dc in transform_rule["destCols"]):
|
||||
errstr = "colIds must be defined in transform_rule for importing into existing table: "
|
||||
raise ValueError(errstr + repr(transform_rule))
|
||||
|
||||
#========= Create new transform view section.
|
||||
new_section = self._docmodel.add(self._docmodel.view_sections,
|
||||
tableRef=src_table_rec.id,
|
||||
parentKey='record',
|
||||
borderWidth=1, defaultWidth=100,
|
||||
sortColRefs='[]')[0]
|
||||
self._docmodel.add(new_section.fields, colRef=new_cols)
|
||||
new_cols = self._MakeImportTransformColumns(source_table_id, transform_rule, gen_all=genAll)
|
||||
|
||||
return new_section.id
|
||||
result = {"transformRule": transform_rule}
|
||||
if createViewSection:
|
||||
#========= Create new transform view section.
|
||||
new_section = self._docmodel.add(self._docmodel.view_sections,
|
||||
tableRef=src_table_rec.id,
|
||||
parentKey='record',
|
||||
borderWidth=1, defaultWidth=100,
|
||||
sortColRefs='[]')[0]
|
||||
self._docmodel.add(new_section.fields, colRef=new_cols)
|
||||
result["viewSectionRef"] = new_section.id
|
||||
|
||||
return result
|
||||
|
||||
@ -66,7 +66,7 @@ class TestImportActions(test_engine.EngineTestCase):
|
||||
|
||||
# Update transform while importing to destination table which have
|
||||
# columns with the same names as source
|
||||
self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None])
|
||||
self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None, {}])
|
||||
|
||||
# Verify the new structure of source table and sections
|
||||
# (two columns with special names were added)
|
||||
@ -98,7 +98,7 @@ class TestImportActions(test_engine.EngineTestCase):
|
||||
|
||||
# Apply useraction again to verify that old columns and sections are removing
|
||||
# Update transform while importing to destination table which has no common columns with source
|
||||
self.apply_user_action(['GenImporterView', 'Source', 'Destination2', None])
|
||||
self.apply_user_action(['GenImporterView', 'Source', 'Destination2', None, {}])
|
||||
|
||||
# Verify the new structure of source table and sections (old special columns were removed
|
||||
# and one new columns with empty formula were added)
|
||||
@ -131,8 +131,8 @@ class TestImportActions(test_engine.EngineTestCase):
|
||||
# Generate without a destination table, and then with one. Ensure that we don't omit the
|
||||
# actions needed to populate the table in the second call.
|
||||
self.init_state()
|
||||
self.apply_user_action(['GenImporterView', 'Source', None, None])
|
||||
out_actions = self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None])
|
||||
self.apply_user_action(['GenImporterView', 'Source', None, None, {}])
|
||||
out_actions = self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None, {}])
|
||||
self.assertPartialOutActions(out_actions, {
|
||||
"stored": [
|
||||
["BulkRemoveRecord", "_grist_Views_section_field", [13, 14, 15]],
|
||||
@ -160,7 +160,7 @@ class TestImportActions(test_engine.EngineTestCase):
|
||||
self.init_state()
|
||||
|
||||
# Update transform while importing to destination table which is "New Table"
|
||||
self.apply_user_action(['GenImporterView', 'Source', None, None])
|
||||
self.apply_user_action(['GenImporterView', 'Source', None, None, {}])
|
||||
|
||||
# Verify the new structure of source table and sections (old special columns were removed
|
||||
# and three new columns, which are the same as in source table were added)
|
||||
|
||||
@ -2134,13 +2134,6 @@ class UserActions(object):
|
||||
#----------------------------------------------------------------------
|
||||
|
||||
@useraction
|
||||
def GenImporterView(self, source_table_id, dest_table_id, transform_rule = None):
|
||||
return self._import_actions.DoGenImporterView(source_table_id, dest_table_id, transform_rule)
|
||||
|
||||
@useraction
|
||||
def MakeImportTransformColumns(self, source_table_id, transform_rule, gen_all):
|
||||
return self._import_actions.MakeImportTransformColumns(source_table_id, transform_rule, gen_all)
|
||||
|
||||
@useraction
|
||||
def FillTransformRuleColIds(self, transform_rule):
|
||||
return self._import_actions.FillTransformRuleColIds(transform_rule)
|
||||
def GenImporterView(self, source_table_id, dest_table_id, transform_rule=None, options=None):
|
||||
return self._import_actions.DoGenImporterView(
|
||||
source_table_id, dest_table_id, transform_rule, options or {})
|
||||
|
||||
BIN
test/fixtures/docs/Exports.grist
vendored
BIN
test/fixtures/docs/Exports.grist
vendored
Binary file not shown.
BIN
test/fixtures/docs/ImportReferences.grist
vendored
BIN
test/fixtures/docs/ImportReferences.grist
vendored
Binary file not shown.
8
test/fixtures/export-csv/reference.csv
vendored
8
test/fixtures/export-csv/reference.csv
vendored
@ -1,4 +1,4 @@
|
||||
Text Bar,Int Text,Text Formula
|
||||
"the ""quote marks"" ?",Integer[500],a --- grist https://www.getgrist.com/
|
||||
"b ,d",Integer[200],"b ,d --- https://www.getgrist.com/"
|
||||
"the ""quote marks"" ?",,a --- grist https://www.getgrist.com/
|
||||
Text Bar,Int Text,Text Formula,RowId
|
||||
"the ""quote marks"" ?",500,a --- grist https://www.getgrist.com/,Text[3]
|
||||
"b ,d",200,"b ,d --- https://www.getgrist.com/",Text[2]
|
||||
"the ""quote marks"" ?",0,a --- grist https://www.getgrist.com/,
|
||||
|
||||
|
BIN
test/fixtures/export-xlsx/Exports.xlsx
vendored
BIN
test/fixtures/export-xlsx/Exports.xlsx
vendored
Binary file not shown.
5
test/fixtures/uploads/ImportReferences-Tasks.csv
vendored
Normal file
5
test/fixtures/uploads/ImportReferences-Tasks.csv
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
Label,PName,PIndex,PIndex2,PDate,PRowID,PID
|
||||
Foo2,Clean,1000,"1,000",27 Mar 2023,,0
|
||||
Bar2,Wash,3000,"2,000",,Projects[2],2
|
||||
Baz2,Build2,,2,20 Mar 2023,Projects[1],1
|
||||
Zoo2,Clean,2000,"4,000",24 Apr 2023,Projects[3],3
|
||||
|
Loading…
Reference in New Issue
Block a user