From 8a7edb625787c33cf4ae401a953b2cc7cdf086a2 Mon Sep 17 00:00:00 2001 From: George Gevoian Date: Tue, 14 Sep 2021 23:12:34 -0700 Subject: [PATCH] (core) Enable incremental imports Summary: The import dialog now has an option to 'Update existing records', which when checked will allow for selection of 1 or more fields to match source and destination tables on. If all fields match, then the matched record in the destination table will be merged with the incoming record from the source table. This means the incoming values will replace the destination table values, unless the incoming values are blank. Additional merge strategies are implemented in the data engine, but the import dialog only uses one of the strategies currently. The others can be exposed in the UI in the future, and tweak the behavior of how source and destination values should be merged in different contexts, such as when blank values exist. Test Plan: Python and browser tests. Reviewers: paulfitz Reviewed By: paulfitz Subscribers: alexmojaki Differential Revision: https://phab.getgrist.com/D3020 --- app/client/components/GristDoc.ts | 2 +- app/client/components/Importer.ts | 142 ++++++- app/client/ui2018/menus.ts | 121 +++++- app/common/ActiveDocAPI.ts | 18 +- app/server/lib/ActiveDoc.ts | 5 +- app/server/lib/ActiveDocImport.ts | 67 +-- sandbox/grist/import_actions.py | 137 ++++++- sandbox/grist/test_import_transform.py | 548 +++++++++++++++++++++---- sandbox/grist/useractions.py | 9 +- 9 files changed, 914 insertions(+), 135 deletions(-) diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index adeb775a..440c455d 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -526,7 +526,7 @@ export class GristDoc extends DisposableWithEvents { multiple: true}); if (uploadResult) { const dataSource = {uploadId: uploadResult.uploadId, transforms: []}; - const importResult = await this.docComm.finishImportFiles(dataSource, {}, []); + const importResult = await this.docComm.finishImportFiles(dataSource, [], {}); const tableId = importResult.tables[0].hiddenTableId; const tableRowModel = this.docModel.dataTables[tableId].tableMetaRow; await this.openDocPage(tableRowModel.primaryViewId()); diff --git a/app/client/components/Importer.ts b/app/client/components/Importer.ts index 83814917..e32f8e18 100644 --- a/app/client/components/Importer.ts +++ b/app/client/components/Importer.ts @@ -15,14 +15,16 @@ import {openFilePicker} from "app/client/ui/FileDialog"; import {bigBasicButton, bigPrimaryButton} from 'app/client/ui2018/buttons'; import {colors, testId, vars} from 'app/client/ui2018/cssVars'; import {icon} from 'app/client/ui2018/icons'; -import {IOptionFull, linkSelect} from 'app/client/ui2018/menus'; +import {IOptionFull, linkSelect, multiSelect} from 'app/client/ui2018/menus'; import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals'; -import {DataSourceTransformed, ImportResult, ImportTableResult} from "app/common/ActiveDocAPI"; -import {TransformColumn, TransformRule, TransformRuleMap} from "app/common/ActiveDocAPI"; +import {DataSourceTransformed, ImportResult, ImportTableResult, MergeOptions, + MergeStrategy, TransformColumn, TransformRule, TransformRuleMap} from "app/common/ActiveDocAPI"; import {byteString} from "app/common/gutil"; import {UploadResult} from 'app/common/uploads'; import {ParseOptions, ParseOptionSchema} from 'app/plugin/FileParserAPI'; -import {Computed, Disposable, dom, DomContents, IDisposable, Observable, styled} from 'grainjs'; +import {Computed, Disposable, dom, DomContents, IDisposable, MutableObsArray, obsArray, Observable, + styled} from 'grainjs'; +import {labeledSquareCheckbox} from "app/client/ui2018/checkbox"; // Special values for import destinations; null means "new table". // TODO We should also support "skip table" (needs server support), so that one can open, say, @@ -45,6 +47,15 @@ export interface SourceInfo { transformSection: Observable; destTableId: Observable; } + // UI state of selected merge options for each source table (from SourceInfo). +interface MergeOptionsState { + [srcTableId: string]: { + updateExistingRecords: Observable; + mergeCols: MutableObsArray; + mergeStrategy: Observable; + hasInvalidMergeCols: Observable; + } | undefined; +} /** * Importer manages an import files to Grist tables and shows Preview @@ -119,6 +130,7 @@ export class Importer extends Disposable { private _uploadResult?: UploadResult; private _screen: PluginScreen; + private _mergeOptions: MergeOptionsState = {}; private _parseOptions = Observable.create(this, {}); private _sourceInfoArray = Observable.create(this, []); private _sourceInfoSelected = Observable.create(this, null); @@ -223,6 +235,22 @@ export class Importer extends Disposable { return {uploadId: upload.uploadId, transforms}; } + private _getMergeOptions(upload: UploadResult): Array { + return upload.files.map((_file, i) => { + const sourceInfo = this._sourceInfoArray.get().find(info => info.uploadFileIndex === i); + if (!sourceInfo) { return null; } + + const mergeOptions = this._mergeOptions[sourceInfo.hiddenTableId]; + if (!mergeOptions) { return null; } + + const {updateExistingRecords, mergeCols, mergeStrategy} = mergeOptions; + return { + mergeCols: updateExistingRecords.get() ? mergeCols.get() : [], + mergeStrategy: mergeStrategy.get() + }; + }); + } + private _createTransformRuleMap(uploadFileIndex: number): TransformRuleMap { const result: TransformRuleMap = {}; for (const sourceInfo of this._sourceInfoArray.get()) { @@ -276,6 +304,16 @@ export class Importer extends Disposable { throw new Error("No data was imported"); } + this._mergeOptions = {}; + this._getHiddenTableIds().forEach(tableId => { + this._mergeOptions[tableId] = { + updateExistingRecords: Observable.create(null, false), + mergeCols: obsArray(), + mergeStrategy: Observable.create(null, {type: 'replace-with-nonblank-source'}), + hasInvalidMergeCols: Observable.create(null, false) + }; + }); + // Select the first sourceInfo to show in preview. this._sourceInfoSelected.set(this._sourceInfoArray.get()[0] || null); @@ -287,11 +325,16 @@ export class Importer extends Disposable { } } - private async _finishImport(upload: UploadResult) { + private async _maybeFinishImport(upload: UploadResult) { + const isConfigValid = this._validateImportConfiguration(); + if (!isConfigValid) { return; } + this._screen.renderSpinner(); const parseOptions = {...this._parseOptions.get(), NUM_ROWS: 0}; + const mergeOptions = this._getMergeOptions(upload); + const importResult: ImportResult = await this._docComm.finishImportFiles( - this._getTransformedDataSource(upload), parseOptions, this._getHiddenTableIds()); + this._getTransformedDataSource(upload), this._getHiddenTableIds(), {mergeOptions, parseOptions}); if (importResult.tables[0].hiddenTableId) { const tableRowModel = this._gristDoc.docModel.dataTables[importResult.tables[0].hiddenTableId].tableMetaRow; @@ -310,6 +353,28 @@ export class Importer extends Disposable { this.dispose(); } + private _resetTableMergeOptions(tableId: string) { + this._mergeOptions[tableId]?.mergeCols.set([]); + } + + private _validateImportConfiguration(): boolean { + let isValid = true; + + const selectedSourceInfo = this._sourceInfoSelected.get(); + if (!selectedSourceInfo) { return isValid; } // No configuration to validate. + + const mergeOptions = this._mergeOptions[selectedSourceInfo.hiddenTableId]; + if (!mergeOptions) { return isValid; } // No configuration to validate. + + const {updateExistingRecords, mergeCols, hasInvalidMergeCols} = mergeOptions; + if (updateExistingRecords.get() && mergeCols.get().length === 0) { + hasInvalidMergeCols.set(true); + isValid = false; + } + + return isValid; + } + private _buildModalTitle(rightElement?: DomContents) { const title = this._importSourceElem ? this._importSourceElem.importSource.label : 'Import from file'; return cssModalHeader(cssModalTitle(title), rightElement); @@ -329,18 +394,64 @@ export class Importer extends Disposable { cssTableList( dom.forEach(this._sourceInfoArray, (info) => { const destTableId = Computed.create(null, (use) => use(info.destTableId)) - .onWrite((destId) => this._updateTransformSection(info, destId)); + .onWrite((destId) => { + this._resetTableMergeOptions(info.hiddenTableId); + void this._updateTransformSection(info, destId); + }); return cssTableInfo( dom.autoDispose(destTableId), cssTableLine(cssToFrom('From'), cssTableSource(getSourceDescription(info, upload), testId('importer-from'))), cssTableLine(cssToFrom('To'), linkSelect(destTableId, this._destTables)), cssTableInfo.cls('-selected', (use) => use(this._sourceInfoSelected) === info), - dom.on('click', () => this._sourceInfoSelected.set(info)), + dom.on('click', () => { + if (info === this._sourceInfoSelected.get() || !this._validateImportConfiguration()) { + return; + } + this._sourceInfoSelected.set(info); + }), testId('importer-source'), ); }), ), + dom.maybe(this._sourceInfoSelected, (info) => + dom.maybe(info.destTableId, () => { + const {mergeCols, updateExistingRecords, hasInvalidMergeCols} = this._mergeOptions[info.hiddenTableId]!; + return cssMergeOptions( + cssMergeOptionsToggle(labeledSquareCheckbox( + updateExistingRecords, + 'Update existing records', + testId('importer-update-existing-records') + )), + dom.maybe(updateExistingRecords, () => [ + cssMergeOptionsMessage( + 'Imported rows will be merged with records that have the same values for all of these fields:', + testId('importer-merge-fields-message') + ), + dom.domComputed(info.transformSection, section => { + // When changes are made to selected fields, reset the multiSelect error observable. + const invalidColsListener = mergeCols.addListener((val, _prev) => { + if (val.length !== 0 && hasInvalidMergeCols.get()) { + hasInvalidMergeCols.set(false); + } + }); + return [ + dom.autoDispose(invalidColsListener), + multiSelect( + mergeCols, + section.viewFields().peek().map(field => field.label()), + { + placeholder: 'Select fields to match on', + error: hasInvalidMergeCols + }, + testId('importer-merge-fields-select') + ), + ]; + }) + ]) + ); + }) + ), dom.maybe(this._previewViewSection, () => cssSectionHeader('Preview')), dom.maybe(this._previewViewSection, (viewSection) => { const gridView = this._createPreview(viewSection); @@ -353,7 +464,7 @@ export class Importer extends Disposable { ), cssModalButtons( bigPrimaryButton('Import', - dom.on('click', () => this._finishImport(upload)), + dom.on('click', () => this._maybeFinishImport(upload)), testId('modal-confirm'), ), bigBasicButton('Cancel', @@ -480,3 +591,16 @@ const cssPreviewGrid = styled('div', ` height: 300px; border: 1px solid ${colors.darkGrey}; `); + +const cssMergeOptions = styled('div', ` + margin-bottom: 16px; +`); + +const cssMergeOptionsToggle = styled('div', ` + margin-bottom: 8px; +`); + +const cssMergeOptionsMessage = styled('div', ` + color: ${colors.slate}; + margin-bottom: 8px; +`); diff --git a/app/client/ui2018/menus.ts b/app/client/ui2018/menus.ts index 322b2aaa..c551ce07 100644 --- a/app/client/ui2018/menus.ts +++ b/app/client/ui2018/menus.ts @@ -5,10 +5,10 @@ import {cssSelectBtn} from 'app/client/ui2018/select'; import {IconName} from 'app/client/ui2018/IconList'; import {icon} from 'app/client/ui2018/icons'; import {commonUrls} from 'app/common/gristUrls'; -import {dom, DomElementArg, DomElementMethod} from 'grainjs'; -import {MaybeObsArray, Observable, styled} from 'grainjs'; +import {Computed, dom, DomElementArg, DomElementMethod, MaybeObsArray, MutableObsArray, Observable, + styled} from 'grainjs'; import * as weasel from 'popweasel'; -import {IAutocompleteOptions} from 'popweasel'; +import {cssCheckboxSquare, cssLabel, cssLabelText} from 'app/client/ui2018/checkbox'; export interface IOptionFull { value: T; @@ -132,6 +132,95 @@ export function linkSelect(obs: Observable, optionArray: MaybeObsArray; +} + +/** + * Creates a select dropdown widget that supports selecting multiple options. + * + * The observable array `selectedOptions` reflects the selected options, and + * `availableOptions` is an array (normal or observable) of selectable options. + * These may either be strings, or {label, value} objects. + */ +export function multiSelect(selectedOptions: MutableObsArray, + availableOptions: MaybeObsArray>, + options: IMultiSelectUserOptions = {}, + ...domArgs: DomElementArg[]) { + const selectedOptionsSet = Computed.create(null, selectedOptions, (_use, opts) => new Set(opts)); + + const selectedOptionsText = Computed.create(null, selectedOptionsSet, (use, selectedOpts) => { + if (selectedOpts.size === 0) { + return options.placeholder ?? 'Select fields'; + } + + const optionArray = Array.isArray(availableOptions) ? availableOptions : use(availableOptions); + return optionArray + .filter(opt => selectedOpts.has(weasel.getOptionFull(opt).value)) + .map(opt => weasel.getOptionFull(opt).label) + .join(', '); + }); + + function buildMultiSelectMenu(ctl: weasel.IOpenController) { + return cssMultiSelectMenu( + { tabindex: '-1' }, // Allow menu to be focused. + dom.cls(menuCssClass), + dom.onKeyDown({ + Enter: () => ctl.close(), + Escape: () => ctl.close() + }), + elem => { + // Set focus on open, so that keyboard events work. + setTimeout(() => elem.focus(), 0); + + // Sets menu width to match parent container (button) width. + const style = elem.style; + style.minWidth = ctl.getTriggerElem().getBoundingClientRect().width + 'px'; + style.marginLeft = style.marginRight = '0'; + }, + dom.domComputed(selectedOptionsSet, selectedOpts => { + return dom.forEach(availableOptions, option => { + const fullOption = weasel.getOptionFull(option); + return cssCheckboxLabel( + cssCheckboxSquare( + {type: 'checkbox'}, + dom.prop('checked', selectedOpts.has(fullOption.value)), + dom.on('change', (_ev, elem) => { + if (elem.checked) { + selectedOptions.push(fullOption.value); + } else { + selectedOpts.delete(fullOption.value); + selectedOptions.set([...selectedOpts]); + } + }), + dom.style('position', 'relative'), + testId('multi-select-menu-option-checkbox') + ), + cssCheckboxText(fullOption.label, testId('multi-select-menu-option-text')), + testId('multi-select-menu-option') + ); + }); + }), + testId('multi-select-menu') + ); + } + + return cssSelectBtn( + dom.autoDispose(selectedOptionsSet), + dom.autoDispose(selectedOptionsText), + cssMultiSelectSummary(dom.text(selectedOptionsText)), + icon('Dropdown'), + elem => { + weasel.setPopupToCreateDom(elem, ctl => buildMultiSelectMenu(ctl), weasel.defaultMenuOptions); + }, + dom.style('border', use => { + return options.error && use(options.error) ? '1px solid red' : `1px solid ${colors.darkGrey}`; + }), + ...domArgs + ); +} + /** * Creates a select dropdown widget that is more ideal for forms. Implemented using the