Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) Add column matching to Importer

Browse Source 
Summary:
The Importer dialog is now maximized, showing additional column
matching options and information on the left, with the preview
table shown on the right. Columns can be mapped via a select menu
listing all source columns, or by clicking a formula field next to
the menu and directly editing the transform formula.

Test Plan: Browser tests.

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D3096
This commit is contained in:
George Gevoian
2021-11-09 12:03:12 -08:00
parent 96fa7ad562
commit 08b1286f4f
11 changed files with 553 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

471
app/client/components/Importer.ts
View File

@@ -4,31 +4,35 @@
*/
// tslint:disable:no-console
import {GristDoc} from "app/client/components/GristDoc";
import {GristDoc} from 'app/client/components/GristDoc';
import {buildParseOptionsForm, ParseOptionValues} from 'app/client/components/ParseOptions';
import {PluginScreen} from "app/client/components/PluginScreen";
import {PluginScreen} from 'app/client/components/PluginScreen';
import {FocusLayer} from 'app/client/lib/FocusLayer';
import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
import {fetchURL, isDriveUrl, selectFiles, uploadFiles} from 'app/client/lib/uploads';
import {reportError} from 'app/client/models/AppModel';
import {ViewSectionRec} from 'app/client/models/DocModel';
import {SortedRowSet} from "app/client/models/rowset";
import {openFilePicker} from "app/client/ui/FileDialog";
import {ColumnRec, ViewFieldRec, ViewSectionRec} from 'app/client/models/DocModel';
import {SortedRowSet} from 'app/client/models/rowset';
import {buildHighlightedCode} from 'app/client/ui/CodeHighlight';
import {openFilePicker} from 'app/client/ui/FileDialog';
import {bigBasicButton, bigPrimaryButton} from 'app/client/ui2018/buttons';
import {colors, testId, vars} from 'app/client/ui2018/cssVars';
import {icon} from 'app/client/ui2018/icons';
import {IOptionFull, linkSelect, multiSelect} from 'app/client/ui2018/menus';
import {IOptionFull, linkSelect, menu, menuDivider, menuItem, multiSelect} from 'app/client/ui2018/menus';
import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals';
import {loadingSpinner} from "app/client/ui2018/loaders";
import {loadingSpinner} from 'app/client/ui2018/loaders';
import {openFormulaEditor} from 'app/client/widgets/FieldEditor';
import {DataSourceTransformed, ImportResult, ImportTableResult, MergeOptions,
MergeOptionsMap,
MergeStrategy, TransformColumn, TransformRule, TransformRuleMap} from "app/common/ActiveDocAPI";
import {byteString} from "app/common/gutil";
MergeStrategy, TransformColumn, TransformRule, TransformRuleMap} from 'app/common/ActiveDocAPI';
import {DisposableWithEvents} from 'app/common/DisposableWithEvents';
import {byteString} from 'app/common/gutil';
import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
import {ParseOptions, ParseOptionSchema} from 'app/plugin/FileParserAPI';
import {Computed, Disposable, dom, DomContents, IDisposable, MutableObsArray, obsArray, Observable,
import {Computed, dom, DomContents, fromKo, Holder, IDisposable, MultiHolder, MutableObsArray, obsArray, Observable,
styled} from 'grainjs';
import {labeledSquareCheckbox} from "app/client/ui2018/checkbox";
import {ACCESS_DENIED, AUTH_INTERRUPTED, canReadPrivateFiles, getGoogleCodeForReading} from "app/client/ui/googleAuth";
import {labeledSquareCheckbox} from 'app/client/ui2018/checkbox';
import {ACCESS_DENIED, AUTH_INTERRUPTED, canReadPrivateFiles, getGoogleCodeForReading} from 'app/client/ui/googleAuth';
import debounce = require('lodash/debounce');
// Special values for import destinations; null means "new table".
@@ -72,7 +76,7 @@ interface MergeOptionsState {
/**
* Importer manages an import files to Grist tables and shows Preview
*/
export class Importer extends Disposable {
export class Importer extends DisposableWithEvents {
/**
* Imports using the given plugin importer, or the built-in file-picker when null is passed in.
*/
@@ -147,6 +151,9 @@ export class Importer extends Disposable {
private _sourceInfoArray = Observable.create<SourceInfo[]>(this, []);
private _sourceInfoSelected = Observable.create<SourceInfo|null>(this, null);
// Holder for the column mapping formula editor.
private readonly _formulaEditorHolder = Holder.create(this);
private _previewViewSection: Observable<ViewSectionRec|null> =
Computed.create(this, this._sourceInfoSelected, (use, info) => {
if (!info) { return null; }
@@ -172,6 +179,27 @@ export class Importer extends Disposable {
...use(this._gristDoc.docModel.allTableIds.getObservable()).map((t) => ({value: t, label: t})),
]);
// Source column labels for the selected import source, keyed by column id.
private _sourceColLabelsById = Computed.create(this, this._sourceInfoSelected, (use, info) => {
if (!info || use(info.sourceSection._isDeleted)) { return null; }
const fields = use(use(info.sourceSection.viewFields).getObservable());
return new Map(fields.map(f => [use(use(f.column).colId), use(use(f.column).label)]));
});
// List of destination fields that aren't mapped to a source column.
private _unmatchedFields = Computed.create(this, this._sourceInfoSelected, (use, info) => {
if (!info) { return null; }
const transformSection = use(info.transformSection);
if (!transformSection || use(transformSection._isDeleted)) { return null; }
const fields = use(use(transformSection.viewFields).getObservable());
return fields
.filter(f => use(use(f.column).formula).trim() === '')
.map(f => f.column().label());
});
// null tells to use the built-in file picker.
constructor(private _gristDoc: GristDoc, private _importSourceElem: ImportSourceElement|null,
private _createPreview: CreatePreviewFunc) {
@@ -406,6 +434,8 @@ export class Importer extends Disposable {
private async _cancelImport() {
this._resetImportDiffState();
// Formula editor cleanup needs to happen before the hidden tables are removed.
this._formulaEditorHolder.dispose();
if (this._uploadResult) {
await this._docComm.cancelImportFiles(
@@ -428,8 +458,11 @@ export class Importer extends Disposable {
const mergeOptions = this._mergeOptions[selectedSourceInfo.hiddenTableId];
if (!mergeOptions) { return isValid; } // No configuration to validate.
const destTableId = selectedSourceInfo.destTableId.get();
const {updateExistingRecords, mergeCols, hasInvalidMergeCols} = mergeOptions;
if (updateExistingRecords.get() && mergeCols.get().length === 0) {
// Check that at least one merge column was selected (if merging into an existing table).
if (destTableId !== null && updateExistingRecords.get() && mergeCols.get().length === 0) {
hasInvalidMergeCols.set(true);
isValid = false;
}
@@ -498,7 +531,9 @@ export class Importer extends Disposable {
// The importer state showing import in progress, with a list of tables, and a preview.
private _renderMain(upload: UploadResult) {
const schema = this._parseOptions.get().SCHEMA;
this._screen.render([
const content = cssContainer(
dom.autoDispose(this._formulaEditorHolder),
{tabIndex: '-1'},
this._buildModalTitle(
schema ? cssActionLink(cssLinkIcon('Settings'), 'Import options',
testId('importer-options-link'),
@@ -510,6 +545,11 @@ export class Importer extends Disposable {
dom.forEach(this._sourceInfoArray, (info) => {
const destTableId = Computed.create(null, (use) => use(info.destTableId))
.onWrite(async (destId) => {
// Prevent changing destination of un-selected sources if current configuration is invalid.
if (info !== this._sourceInfoSelected.get() && !this._validateImportConfiguration()) {
return;
}
info.destTableId.set(destId);
this._resetTableMergeOptions(info.hiddenTableId);
await this._updateTransformSection(info);
@@ -521,9 +561,11 @@ export class Importer extends Disposable {
cssTableLine(cssToFrom('To'), linkSelect<DestId>(destTableId, this._destTables)),
cssTableInfo.cls('-selected', (use) => use(this._sourceInfoSelected) === info),
dom.on('click', async () => {
if (info === this._sourceInfoSelected.get() || !this._validateImportConfiguration()) {
return;
}
// Ignore click if source is already selected.
if (info === this._sourceInfoSelected.get()) { return; }
// Prevent changing selected source if current configuration is invalid.
if (!this._validateImportConfiguration()) { return; }
this._cancelPendingDiffRequests();
this._sourceInfoSelected.set(info);
@@ -536,68 +578,141 @@ export class Importer extends Disposable {
dom.maybe(this._sourceInfoSelected, (info) => {
const {mergeCols, updateExistingRecords, hasInvalidMergeCols} = this._mergeOptions[info.hiddenTableId]!;
return [
dom.maybe(info.destTableId, (_dest) => {
const updateRecordsListener = updateExistingRecords.addListener(async () => {
await this._updateImportDiff(info);
});
return cssConfigAndPreview(
cssConfigColumn(
dom.maybe(info.transformSection, section => [
dom.maybe(info.destTableId, () => {
const updateRecordsListener = updateExistingRecords.addListener(async () => {
await this._updateImportDiff(info);
});
return cssMergeOptions(
cssMergeOptionsToggle(labeledSquareCheckbox(
updateExistingRecords,
'Update existing records',
dom.autoDispose(updateRecordsListener),
testId('importer-update-existing-records')
)),
dom.maybe(updateExistingRecords, () => [
cssMergeOptionsMessage(
'Imported rows will be merged with records that have the same values for all of these fields:',
testId('importer-merge-fields-message')
),
dom.domComputed(info.transformSection, section => {
const mergeColsListener = mergeCols.addListener(async val => {
// Reset the error state of the multiSelect on change.
if (val.length !== 0 && hasInvalidMergeCols.get()) {
hasInvalidMergeCols.set(false);
}
return cssMergeOptions(
cssMergeOptionsToggle(labeledSquareCheckbox(
updateExistingRecords,
'Update existing records',
dom.autoDispose(updateRecordsListener),
testId('importer-update-existing-records')
)),
dom.maybe(updateExistingRecords, () => {
const mergeColsListener = mergeCols.addListener(async val => {
// Reset the error state of the multiSelect on change.
if (val.length !== 0 && hasInvalidMergeCols.get()) {
hasInvalidMergeCols.set(false);
}
await this._updateImportDiff(info);
});
await this._updateImportDiff(info);
});
return multiSelect(
mergeCols,
section?.viewFields().peek().map(field => field.label()) ?? [],
{
placeholder: 'Select fields to match on',
error: hasInvalidMergeCols
},
dom.autoDispose(mergeColsListener),
testId('importer-merge-fields-select')
);
})
])
);
}),
cssSectionHeader('Preview'),
dom.domComputed(use => {
const previewSection = use(this._previewViewSection);
if (use(this._isLoadingDiff) || !previewSection) {
return cssPreviewSpinner(loadingSpinner());
}
return [
cssMergeOptionsMessage(
'Merge rows that match these fields:',
testId('importer-merge-fields-message')
),
multiSelect(
mergeCols,
section.viewFields().peek().map(f => ({label: f.label(), value: f.colId()})) ?? [],
{
placeholder: 'Select fields to match on',
error: hasInvalidMergeCols
},
dom.autoDispose(mergeColsListener),
testId('importer-merge-fields-select')
)
];
})
);
}),
dom.domComputed(this._unmatchedFields, fields =>
fields && fields.length > 0 ?
cssUnmatchedFields(
dom('div',
cssGreenText(
`${fields.length} unmatched ${fields.length > 1 ? 'fields' : 'field'}`
),
' in import:'
),
cssUnmatchedFieldsList(fields.join(', ')),
testId('importer-unmatched-fields')
) : null
),
cssColumnMatchOptions(
dom.forEach(fromKo(section.viewFields().getObservable()), field => cssColumnMatchRow(
cssColumnMatchIcon('ImportArrow'),
cssSourceAndDestination(
cssDestinationFieldRow(
cssDestinationFieldLabel(
dom.text(field.label),
),
cssDestinationFieldSettings(
icon('Dots'),
menu(
() => {
const sourceColId = field.origCol().id();
const sourceColIdsAndLabels = [...this._sourceColLabelsById.get()!.entries()];
return [
menuItem(
() => this._gristDoc.clearColumns([sourceColId]),
'Skip',
testId('importer-column-match-menu-item')
),
menuDivider(),
...sourceColIdsAndLabels.map(([id, label]) =>
menuItem(
() => this._setColumnFormula(sourceColId, '$' + id),
label,
testId('importer-column-match-menu-item')
),
),
testId('importer-column-match-menu'),
];
},
{placement: 'right-start'},
),
testId('importer-column-match-destination-settings')
),
testId('importer-column-match-destination')
),
dom.domComputed(use => dom.create(
this._buildColMappingFormula.bind(this),
use(field.column),
(elem: Element) => this._activateFormulaEditor(elem, field),
'Skip'
)),
testId('importer-column-match-source-destination'),
)
)),
testId('importer-column-match-options'),
)
]),
),
cssPreviewColumn(
cssSectionHeader('Preview'),
dom.domComputed(use => {
const previewSection = use(this._previewViewSection);
if (use(this._isLoadingDiff) || !previewSection) {
return cssPreviewSpinner(loadingSpinner());
}
const gridView = this._createPreview(previewSection);
const gridView = this._createPreview(previewSection);
// When changes are made to the preview table, update the import diff.
gridView.listenTo(gridView.sortedRows, 'rowNotify', async () => {
await this._updateImportDiff(info);
});
// When changes are made to the preview table, update the import diff.
gridView.listenTo(gridView.sortedRows, 'rowNotify', async () => {
// If we aren't showing a diff, there is no need to do anything.
if (!info.destTableId || !updateExistingRecords.get() || mergeCols.get().length === 0) {
return;
}
return cssPreviewGrid(
dom.autoDispose(gridView),
gridView.viewPane,
testId('importer-preview'),
);
})
];
// Otherwise, update the diff and rebuild the preview table.
await this._updateImportDiff(info);
});
return cssPreviewGrid(
dom.autoDispose(gridView),
gridView.viewPane,
testId('importer-preview'),
);
})
)
);
}),
),
cssModalButtons(
@@ -611,7 +726,88 @@ export class Importer extends Disposable {
testId('modal-cancel'),
),
),
]);
);
this._addFocusLayer(content);
this._screen.render(content, {fullscreen: true});
}
private _addFocusLayer(container: HTMLElement) {
dom.autoDisposeElem(container, new FocusLayer({
defaultFocusElem: container,
allowFocus: (elem) => (elem !== document.body),
onDefaultFocus: () => this.trigger('importer_focus'),
}));
}
/**
* Updates the formula on column `colRef` to `formula`.
*/
private async _setColumnFormula(colRef: number, formula: string): Promise<void> {
return this._gristDoc.docModel.columns.sendTableAction(
['UpdateRecord', colRef, { formula, isFormula: true }]
);
}
/**
* Opens a formula editor for `field` over `refElem`.
*/
private _activateFormulaEditor(refElem: Element, field: ViewFieldRec) {
// TODO: Set active section to hidden table section, so editor autocomplete is accurate.
const editorHolder = openFormulaEditor({
gristDoc: this._gristDoc,
field,
refElem,
setupCleanup: this._setupFormulaEditorCleanup.bind(this),
});
this._formulaEditorHolder.autoDispose(editorHolder);
}
/**
* Called by _activateFormulaEditor to initialize cleanup
* code for when the formula editor is closed. Registers and
* unregisters callbacks for saving edits when the editor loses
* focus.
*/
private _setupFormulaEditorCleanup(
owner: MultiHolder, _doc: GristDoc, field: ViewFieldRec, _saveEdit: () => Promise<unknown>
) {
const saveEdit = () => _saveEdit().catch(reportError);
// Whenever focus returns to the dialog, close the editor by saving the value.
this.on('importer_focus', saveEdit);
owner.onDispose(() => {
this.off('importer_focus', saveEdit);
field.editingFormula(false);
});
}
/**
* Builds an editable formula component that is displayed
* in the column mapping section of Importer. On click, opens
* an editor for the formula for `column`.
*/
private _buildColMappingFormula(_owner: MultiHolder, column: ColumnRec, buildEditor: (e: Element) => void,
placeholder: string) {
const formatFormula = (formula: string) => {
const sourceColLabels = this._sourceColLabelsById.get();
if (!sourceColLabels) { return formula; }
formula = formula.trim();
if (formula.startsWith('$') && sourceColLabels.has(formula.slice(1))) {
// For simple formulas that only reference a source column id, show the source column label.
return sourceColLabels.get(formula.slice(1))!;
}
return formula;
};
return cssFieldFormula(use => formatFormula(use(column.formula)), {placeholder, maxLines: 1},
dom.cls('disabled'),
{tabIndex: '-1'},
dom.on('focus', (_ev, elem) => buildEditor(elem)),
testId('importer-column-match-formula'),
);
}
// The importer state showing parse options that may be changed.
@@ -681,6 +877,13 @@ function getSourceDescription(sourceInfo: SourceInfo, upload: UploadResult) {
return sourceInfo.origTableName ? origName + ' - ' + sourceInfo.origTableName : origName;
}
const cssContainer = styled('div', `
height: 100%;
display: flex;
flex-direction: column;
outline: unset;
`);
const cssActionLink = styled('div', `
display: inline-flex;
align-items: center;
@@ -709,8 +912,9 @@ const cssModalHeader = styled('div', `
`);
const cssPreviewWrapper = styled('div', `
width: 600px;
padding: 8px 12px 8px 0;
display: flex;
flex-direction: column;
flex-grow: 1;
overflow-y: auto;
`);
@@ -725,17 +929,19 @@ const cssSectionHeader = styled('div', `
`);
const cssTableList = styled('div', `
max-height: 50%;
column-gap: 32px;
display: flex;
flex-flow: row wrap;
justify-content: space-between;
margin-bottom: 16px;
align-items: flex-start;
overflow-y: auto;
`);
const cssTableInfo = styled('div', `
padding: 4px 8px;
margin: 4px 0px;
width: calc(50% - 16px);
width: 300px;
border-radius: 3px;
border: 1px solid ${colors.darkGrey};
&:hover, &-selected {
@@ -762,12 +968,35 @@ const cssToFrom = styled('span', `
`);
const cssTableSource = styled('div', `
overflow-wrap: anywhere;
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
`);
const cssConfigAndPreview = styled('div', `
display: flex;
gap: 32px;
flex-grow: 1;
height: 0px;
`);
const cssConfigColumn = styled('div', `
width: 300px;
padding-right: 8px;
display: flex;
flex-direction: column;
overflow-y: auto;
`);
const cssPreviewColumn = styled('div', `
display: flex;
flex-direction: column;
flex-grow: 1;
`);
const cssPreview = styled('div', `
display: flex;
height: 300px;
flex-grow: 1;
`);
const cssPreviewSpinner = styled(cssPreview, `
@@ -791,3 +1020,81 @@ const cssMergeOptionsMessage = styled('div', `
color: ${colors.slate};
margin-bottom: 8px;
`);
const cssColumnMatchOptions = styled('div', `
display: flex;
flex-direction: column;
gap: 20px;
`);
const cssColumnMatchRow = styled('div', `
display: flex;
align-items: center;
`);
const cssFieldFormula = styled(buildHighlightedCode, `
flex: auto;
cursor: pointer;
margin-top: 1px;
padding-left: 4px;
--icon-color: ${colors.lightGreen};
`);
const cssColumnMatchIcon = styled(icon, `
flex-shrink: 0;
width: 20px;
height: 32px;
background-color: ${colors.darkGrey};
margin-right: 4px;
`);
const cssDestinationFieldRow = styled('div', `
align-items: center;
display: flex;
`);
const cssSourceAndDestination = styled('div', `
min-width: 0;
display: flex;
flex-direction: column;
flex-grow: 1;
`);
const cssDestinationFieldLabel = styled('div', `
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
padding-left: 4px;
`);
const cssDestinationFieldSettings = styled('div', `
flex: none;
margin: 0 4px 0 auto;
height: 24px;
width: 24px;
padding: 4px;
line-height: 0px;
border-radius: 3px;
cursor: pointer;
--icon-color: ${colors.slate};
&:hover, &.weasel-popup-open {
background-color: ${colors.mediumGrey};
}
`);
const cssUnmatchedFields = styled('div', `
margin-bottom: 16px;
`);
const cssUnmatchedFieldsList = styled('div', `
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
padding-right: 16px;
color: ${colors.slate};
`);
const cssGreenText = styled('span', `
color: ${colors.lightGreen};
`);

18
app/client/components/PluginScreen.ts
View File

@@ -6,12 +6,21 @@ import { PluginInstance } from 'app/common/PluginInstance';
import { RenderTarget } from 'app/plugin/RenderOptions';
import { Disposable, dom, DomContents, Observable, styled } from 'grainjs';
/**
* Rendering options for the PluginScreen modal.
*/
export interface RenderOptions {
// Maximizes modal to fill the viewport.
fullscreen?: boolean;
}
/**
* Helper for showing plugin components during imports.
*/
export class PluginScreen extends Disposable {
private _openModalCtl: IModalControl | null = null;
private _importerContent = Observable.create<DomContents>(this, null);
private _fullscreen = Observable.create(this, false);
constructor(private _title: string) {
super();
@@ -33,9 +42,10 @@ export class PluginScreen extends Disposable {
return handle;
}
public render(content: DomContents) {
public render(content: DomContents, options?: RenderOptions) {
this.showImportDialog();
this._importerContent.set(content);
this._fullscreen.set(Boolean(options?.fullscreen));
}
// The importer state showing just an error.
@@ -67,6 +77,7 @@ export class PluginScreen extends Disposable {
this._openModalCtl = ctl;
return [
cssModalOverrides.cls(''),
cssModalOverrides.cls('-fullscreen', this._fullscreen),
dom.domComputed(this._importerContent),
testId('importer-dialog'),
];
@@ -89,6 +100,11 @@ const cssModalOverrides = styled('div', `
& > .${cssModalButtons.className} {
margin-top: 16px;
}
&-fullscreen {
height: 100%;
margin: 32px;
}
`);
const cssModalBody = styled('div', `

2
app/client/ui2018/IconList.ts
View File

@@ -56,6 +56,7 @@ export type IconName = "ChartArea" |
"Home" |
"Idea" |
"Import" |
"ImportArrow" |
"Info" |
"LeftAlign" |
"Lock" |
@@ -153,6 +154,7 @@ export const IconList: IconName[] = ["ChartArea",
"Home",
"Idea",
"Import",
"ImportArrow",
"Info",
"LeftAlign",
"Lock",

9
app/client/ui2018/menus.ts
View File

@@ -209,7 +209,10 @@ export function multiSelect<T>(selectedOptions: MutableObsArray<T>,
return cssSelectBtn(
dom.autoDispose(selectedOptionsSet),
dom.autoDispose(selectedOptionsText),
cssMultiSelectSummary(dom.text(selectedOptionsText)),
cssMultiSelectSummary(
dom.text(selectedOptionsText),
cssMultiSelectSummary.cls('-placeholder', use => use(selectedOptionsSet).size === 0)
),
icon('Dropdown'),
elem => {
weasel.setPopupToCreateDom(elem, ctl => buildMultiSelectMenu(ctl), weasel.defaultMenuOptions);
@@ -535,6 +538,10 @@ const cssMultiSelectSummary = styled('div', `
flex: 1 1 0px;
overflow: hidden;
text-overflow: ellipsis;
&-placeholder {
color: ${colors.slate}
}
`);
const cssMultiSelectMenu = styled(weasel.cssMenu, `

5
app/client/widgets/FieldBuilder.ts
View File

@@ -19,7 +19,7 @@ import { buttonSelect } from 'app/client/ui2018/buttonSelect';
import { IOptionFull, menu, select } from 'app/client/ui2018/menus';
import { DiffBox } from 'app/client/widgets/DiffBox';
import { buildErrorDom } from 'app/client/widgets/ErrorDom';
import { FieldEditor, openSideFormulaEditor, saveWithoutEditor } from 'app/client/widgets/FieldEditor';
import { FieldEditor, openFormulaEditor, saveWithoutEditor, setupEditorCleanup } from 'app/client/widgets/FieldEditor';
import { NewAbstractWidget } from 'app/client/widgets/NewAbstractWidget';
import { NewBaseEditor } from "app/client/widgets/NewBaseEditor";
import * as UserType from 'app/client/widgets/UserType';
@@ -523,9 +523,10 @@ export class FieldBuilder extends Disposable {
editValue?: string,
onSave?: (formula: string) => Promise<void>,
onCancel?: () => void) {
const editorHolder = openSideFormulaEditor({
const editorHolder = openFormulaEditor({
gristDoc: this.gristDoc,
field: this.field,
setupCleanup: setupEditorCleanup,
editRow,
refElem,
editValue,

27
app/client/widgets/FieldEditor.ts
View File

@@ -15,7 +15,7 @@ import {isRaisedException} from 'app/common/gristTypes';
import * as gutil from 'app/common/gutil';
import {Disposable, Emitter, Holder, MultiHolder, Observable} from 'grainjs';
import isEqual = require('lodash/isEqual');
import { CellPosition } from "app/client/components/CellPosition";
import {CellPosition} from "app/client/components/CellPosition";
type IEditorConstructor = typeof NewBaseEditor;
@@ -373,18 +373,27 @@ export class FieldEditor extends Disposable {
}
/**
* Open a formula editor in the side pane. Returns a Disposable that owns the editor.
* Open a formula editor. Returns a Disposable that owns the editor.
*/
export function openSideFormulaEditor(options: {
export function openFormulaEditor(options: {
gristDoc: GristDoc,
field: ViewFieldRec,
editRow: DataRowModel, // Needed to get exception value, if any.
refElem: Element, // Element in the side pane over which to position the editor.
// Needed to get exception value, if any.
editRow?: DataRowModel,
// Element over which to position the editor.
refElem: Element,
editValue?: string,
onSave?: (formula: string) => Promise<void>,
onCancel?: () => void,
// Called after editor is created to set up editor cleanup (e.g. saving on click-away).
setupCleanup: (
owner: MultiHolder,
doc: GristDoc,
field: ViewFieldRec,
save: () => Promise<void>
) => void,
}): Disposable {
const {gristDoc, field, editRow, refElem} = options;
const {gristDoc, field, editRow, refElem, setupCleanup} = options;
const holder = MultiHolder.create(null);
const column = field.column();
@@ -411,7 +420,7 @@ export function openSideFormulaEditor(options: {
gristDoc,
field,
cellValue: column.formula(),
formulaError: getFormulaError(gristDoc, editRow, column),
formulaError: editRow ? getFormulaError(gristDoc, editRow, column) : undefined,
editValue: options.editValue,
cursorPos: Number.POSITIVE_INFINITY, // Position of the caret within the editor.
commands: editCommands,
@@ -422,7 +431,7 @@ export function openSideFormulaEditor(options: {
// Enter formula-editing mode (highlight formula icons; click on a column inserts its ID).
field.editingFormula(true);
setupEditorCleanup(holder, gristDoc, field, saveEdit);
setupCleanup(holder, gristDoc, field, saveEdit);
return holder;
}
@@ -447,7 +456,7 @@ function setupReadonlyEditorCleanup(
* - unset field.editingFormula mode
* - Arrange for UnsavedChange protection against leaving the page with unsaved changes.
*/
function setupEditorCleanup(
export function setupEditorCleanup(
owner: MultiHolder, gristDoc: GristDoc, field: ViewFieldRec, _saveEdit: () => Promise<unknown>
) {
const saveEdit = () => _saveEdit().catch(reportError);

78
app/server/lib/ActiveDocImport.ts
View File

@@ -18,6 +18,7 @@ import {DocSession, OptDocSession} from 'app/server/lib/DocSession';
import * as log from 'app/server/lib/log';
import {globalUploadSet, moveUpload, UploadInfo} from 'app/server/lib/uploads';
import {buildComparisonQuery} from 'app/server/lib/ExpandedQuery';
import flatten = require('lodash/flatten');
const IMPORT_TRANSFORM_COLUMN_PREFIX = 'gristHelper_Import_';
@@ -131,16 +132,19 @@ export class ActiveDocImport {
*/
public async generateImportDiff(hiddenTableId: string, {destCols, destTableId}: TransformRule,
{mergeCols, mergeStrategy}: MergeOptions): Promise<DocStateComparison> {
// Merge column ids from client have prefixes that need to be stripped.
mergeCols = stripPrefixes(mergeCols);
// Get column differences between `hiddenTableId` and `destTableId` for rows that exist in both tables.
const selectColumns: [string, string][] =
destCols.map(c => [c.colId!, c.colId!.slice(IMPORT_TRANSFORM_COLUMN_PREFIX.length)]);
const selectColumnsMap = new Map(selectColumns);
const comparisonResult = await this._getTableComparison(hiddenTableId, destTableId!, selectColumnsMap, mergeCols);
const srcAndDestColIds: [string, string[]][] =
destCols.map(c => [c.colId!, [c.colId!.slice(IMPORT_TRANSFORM_COLUMN_PREFIX.length)]]);
const srcToDestColIds = new Map(srcAndDestColIds);
const comparisonResult = await this._getTableComparison(hiddenTableId, destTableId!, srcToDestColIds, mergeCols);
// Initialize container for updated column values in the expected format (ColumnDelta).
const updatedRecords: {[colId: string]: ColumnDelta} = {};
const updatedRecordIds: number[] = [];
const srcColIds = selectColumns.map(([srcColId, _destColId]) => srcColId);
const srcColIds = srcAndDestColIds.map(([srcColId, _destColId]) => srcColId);
for (const id of srcColIds) {
updatedRecords[id] = {};
}
@@ -160,7 +164,7 @@ export class ActiveDocImport {
} else {
// Otherwise, a match was found between source and destination tables.
for (const srcColId of srcColIds) {
const matchingDestColId = selectColumnsMap.get(srcColId);
const matchingDestColId = srcToDestColIds.get(srcColId)![0];
const srcVal = comparisonResult[`${hiddenTableId}.${srcColId}`][i];
const destVal = comparisonResult[`${destTableId}.${matchingDestColId}`][i];
@@ -382,7 +386,7 @@ export class ActiveDocImport {
}
// Transform rules from client may have prefixed column ids, so we need to strip them.
stripPrefixes(transformRule);
stripRulePrefixes(transformRule);
if (intoNewTable) {
// Transform rules for new tables don't have filled in destination column ids.
@@ -444,15 +448,25 @@ export class ActiveDocImport {
private async _mergeAndFinishImport(docSession: OptDocSession, hiddenTableId: string, destTableId: string,
{destCols, sourceCols}: TransformRule,
{mergeCols, mergeStrategy}: MergeOptions): Promise<void> {
// Merge column ids from client have prefixes that need to be stripped.
mergeCols = stripPrefixes(mergeCols);
// Get column differences between `hiddenTableId` and `destTableId` for rows that exist in both tables.
const selectColumns: [string, string][] = destCols.map(destCol => {
const srcAndDestColIds: [string, string][] = destCols.map(destCol => {
const formula = destCol.formula.trim();
const srcColId = formula.startsWith('$') && sourceCols.includes(formula.slice(1)) ?
formula.slice(1) : IMPORT_TRANSFORM_COLUMN_PREFIX + destCol.colId;
return [srcColId, destCol.colId!];
});
const selectColumnsMap = new Map(selectColumns);
const comparisonResult = await this._getTableComparison(hiddenTableId, destTableId, selectColumnsMap, mergeCols);
const srcToDestColIds: Map<string, string[]> = new Map();
srcAndDestColIds.forEach(([srcColId, destColId]) => {
if (!srcToDestColIds.has(srcColId)) {
srcToDestColIds.set(srcColId, [destColId]);
} else {
srcToDestColIds.get(srcColId)!.push(destColId);
}
});
const comparisonResult = await this._getTableComparison(hiddenTableId, destTableId, srcToDestColIds, mergeCols);
// Initialize containers for new and updated records in the expected formats.
const newRecords: BulkColValues = {};
@@ -460,7 +474,7 @@ export class ActiveDocImport {
const updatedRecords: BulkColValues = {};
const updatedRecordIds: number[] = [];
const destColIds = [...selectColumnsMap.values()];
const destColIds = flatten([...srcToDestColIds.values()]);
for (const id of destColIds) {
newRecords[id] = [];
updatedRecords[id] = [];
@@ -469,23 +483,27 @@ export class ActiveDocImport {
// Retrieve the function used to reconcile differences between source and destination.
const merge = getMergeFunction(mergeStrategy);
const srcColIds = [...selectColumnsMap.keys()];
const srcColIds = [...srcToDestColIds.keys()];
const numResultRows = comparisonResult[hiddenTableId + '.id'].length;
for (let i = 0; i < numResultRows; i++) {
if (comparisonResult[destTableId + '.id'][i] === null) {
// No match in destination table found for source row, so it must be a new record.
for (const srcColId of srcColIds) {
const matchingDestColId = selectColumnsMap.get(srcColId);
newRecords[matchingDestColId!].push(comparisonResult[`${hiddenTableId}.${srcColId}`][i]);
const matchingDestColIds = srcToDestColIds.get(srcColId);
matchingDestColIds!.forEach(id => {
newRecords[id].push(comparisonResult[`${hiddenTableId}.${srcColId}`][i]);
});
}
numNewRecords++;
} else {
// Otherwise, a match was found between source and destination tables, so we merge their columns.
for (const srcColId of srcColIds) {
const matchingDestColId = selectColumnsMap.get(srcColId);
const matchingDestColIds = srcToDestColIds.get(srcColId);
const srcVal = comparisonResult[`${hiddenTableId}.${srcColId}`][i];
const destVal = comparisonResult[`${destTableId}.${matchingDestColId}`][i];
updatedRecords[matchingDestColId!].push(merge(srcVal, destVal));
matchingDestColIds!.forEach(id => {
const destVal = comparisonResult[`${destTableId}.${id}`][i];
updatedRecords[id].push(merge(srcVal, destVal));
});
}
updatedRecordIds.push(comparisonResult[destTableId + '.id'][i] as number);
}
@@ -515,17 +533,23 @@ export class ActiveDocImport {
*
* @param {string} hiddenTableId Source table.
* @param {string} destTableId Destination table.
* @param {string} selectColumnsMap Map of source to destination column ids to include in the comparison results.
* @param {Map<string, string[]>} srcToDestColIds Map of source to one or more destination column ids
* to include in the comparison results.
* @param {string[]} mergeCols List of (destination) column ids to use for matching.
* @returns {Promise<BulkColValues} Decoded column values from both tables that were matched, and had differences.
*/
private async _getTableComparison(hiddenTableId: string, destTableId: string, selectColumnsMap: Map<string, string>,
private async _getTableComparison(hiddenTableId: string, destTableId: string, srcToDestColIds: Map<string, string[]>,
mergeCols: string[]): Promise<BulkColValues> {
const joinColumns: [string, string][] =
[...selectColumnsMap.entries()].filter(([_srcColId, destColId]) => mergeCols.includes(destColId));
const joinColumnsMap = new Map(joinColumns);
const mergeColIds = new Set(mergeCols);
const destToSrcMergeColIds = new Map();
srcToDestColIds.forEach((destColIds, srcColId) => {
const maybeMergeColId = destColIds.find(colId => mergeColIds.has(colId));
if (maybeMergeColId !== undefined) {
destToSrcMergeColIds.set(maybeMergeColId, srcColId);
}
});
const query = buildComparisonQuery(hiddenTableId, destTableId, selectColumnsMap, joinColumnsMap);
const query = buildComparisonQuery(hiddenTableId, destTableId, srcToDestColIds, destToSrcMergeColIds);
const result = await this._activeDoc.docStorage.fetchQuery(query);
return this._activeDoc.docStorage.decodeMarshalledDataFromTables(result);
}
@@ -636,7 +660,7 @@ function isBlank(value: CellValue): boolean {
}
// Helper function that strips import prefixes from columns in transform rules (if ids are present).
function stripPrefixes({destCols}: TransformRule): void {
function stripRulePrefixes({destCols}: TransformRule): void {
for (const col of destCols) {
const colId = col.colId;
if (colId && colId.startsWith(IMPORT_TRANSFORM_COLUMN_PREFIX)) {
@@ -645,6 +669,12 @@ function stripPrefixes({destCols}: TransformRule): void {
}
}
// Helper function that returns new `colIds` with import prefixes stripped.
function stripPrefixes(colIds: string[]): string[] {
return colIds.map(id => id.startsWith(IMPORT_TRANSFORM_COLUMN_PREFIX) ?
id.slice(IMPORT_TRANSFORM_COLUMN_PREFIX.length) : id);
}
type MergeFunction = (srcVal: CellValue, destVal: CellValue) => CellValue;
/**

53
app/server/lib/ExpandedQuery.ts
View File

@@ -148,13 +148,15 @@ export function expandQuery(iquery: ServerQuery, docData: DocData, onDemandFormu
*
* @param {string} leftTableId Name of the left table in the comparison.
* @param {string} rightTableId Name of the right table in the comparison.
* @param {Map<string, string>} selectColumns Map of left table column ids to their matching equivalent
* from the right table. All of these columns will be included in the result, aliased by table id.
* @param {Map<string, string>} joinColumns Map of left table column ids to their matching equivalent
* from the right table. These columns are used to join `leftTableID` to `rightTableId`.
* @param {Map<string, string[]>} selectColumns Map of left table column ids to their matching equivalent(s)
* from the right table. A single left column can be compared against 2 or more right columns, so the
* values of `selectColumns` are arrays. All of these columns will be included in the result, aliased by
* table id.
* @param {Map<string, string>} joinColumns Map of right table column ids to their matching equivalent
* from the left table. These columns are used to join `leftTableId` to `rightTableId`.
* @returns {ExpandedQuery} The constructed query.
*/
export function buildComparisonQuery(leftTableId: string, rightTableId: string, selectColumns: Map<string, string>,
export function buildComparisonQuery(leftTableId: string, rightTableId: string, selectColumns: Map<string, string[]>,
joinColumns: Map<string, string>): ExpandedQuery {
const query: ExpandedQuery = { tableId: leftTableId, filters: {} };
@@ -169,14 +171,16 @@ export function buildComparisonQuery(leftTableId: string, rightTableId: string,
`${quoteIdent(rightTableId)}.id AS ${quoteIdent(rightTableId + '.id')}`
);
// Select columns from both tables using the table id as a prefix for each column name.
selectColumns.forEach((rightTableColumn, leftTableColumn) => {
// Select columns from both tables, using the table id as a prefix for each column name.
selectColumns.forEach((rightTableColumns, leftTableColumn) => {
const leftColumnAlias = `${leftTableId}.${leftTableColumn}`;
const rightColumnAlias = `${rightTableId}.${rightTableColumn}`;
selects.push(
`${quoteIdent(leftTableId)}.${quoteIdent(leftTableColumn)} AS ${quoteIdent(leftColumnAlias)}`,
`${quoteIdent(rightTableId)}.${quoteIdent(rightTableColumn)} AS ${quoteIdent(rightColumnAlias)}`
);
selects.push(`${quoteIdent(leftTableId)}.${quoteIdent(leftTableColumn)} AS ${quoteIdent(leftColumnAlias)}`);
rightTableColumns.forEach(colId => {
const rightColumnAlias = `${rightTableId}.${colId}`;
selects.push(`${quoteIdent(rightTableId)}.${quoteIdent(colId)} AS ${quoteIdent(rightColumnAlias)}`
);
});
});
/**
@@ -189,14 +193,14 @@ export function buildComparisonQuery(leftTableId: string, rightTableId: string,
* the left table can only be matched with at most 1 equivalent row from the right table.
*/
const dedupedRightTableQuery =
`SELECT MIN(id) AS id, ${[...joinColumns.values()].map(v => quoteIdent(v)).join(', ')} ` +
`SELECT MIN(id) AS id, ${[...joinColumns.keys()].map(v => quoteIdent(v)).join(', ')} ` +
`FROM ${quoteIdent(rightTableId)} ` +
`GROUP BY ${[...joinColumns.values()].map(v => quoteIdent(v)).join(', ')}`;
`GROUP BY ${[...joinColumns.keys()].map(v => quoteIdent(v)).join(', ')}`;
const dedupedRightTableAlias = quoteIdent('deduped_' + rightTableId);
// Join the left table to the (de-duplicated) right table, and include unmatched left rows.
const joinConditions: string[] = [];
joinColumns.forEach((rightTableColumn, leftTableColumn) => {
joinColumns.forEach((leftTableColumn, rightTableColumn) => {
const leftExpression = `${quoteIdent(leftTableId)}.${quoteIdent(leftTableColumn)}`;
const rightExpression = `${dedupedRightTableAlias}.${quoteIdent(rightTableColumn)}`;
joinConditions.push(`${leftExpression} = ${rightExpression}`);
@@ -212,16 +216,21 @@ export function buildComparisonQuery(leftTableId: string, rightTableId: string,
// Filter out matching rows where all non-join columns from both tables are identical.
const whereConditions: string[] = [];
for (const [leftTableColumn, rightTableColumn] of selectColumns.entries()) {
if (joinColumns.has(leftTableColumn)) { continue; }
for (const [leftTableColumnId, rightTableColumnIds] of selectColumns.entries()) {
const leftColumnAlias = quoteIdent(`${leftTableId}.${leftTableColumnId}`);
const leftColumnAlias = quoteIdent(`${leftTableId}.${leftTableColumn}`);
const rightColumnAlias = quoteIdent(`${rightTableId}.${rightTableColumn}`);
for (const rightTableColId of rightTableColumnIds) {
// If this left/right column id pair was already used for joining, skip it.
if (joinColumns.get(rightTableColId) === leftTableColumnId) { continue; }
// Only include rows that have differences in column values.
whereConditions.push(`${leftColumnAlias} IS NOT ${rightColumnAlias}`);
// Only include rows that have differences in column values.
const rightColumnAlias = quoteIdent(`${rightTableId}.${rightTableColId}`);
whereConditions.push(`${leftColumnAlias} IS NOT ${rightColumnAlias}`);
}
}
if (whereConditions.length > 0) {
wheres.push(`(${whereConditions.join(' OR ')})`);
}
wheres.push(`(${whereConditions.join(' OR ')})`);
// Copy decisions to the query object, and return.
query.joins = joins;