Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) Migrate import code from data engine to Node

Browse Source 
Summary:
Finishing imports now occurs in Node instead of the
data engine, which makes it possible to import into
on-demand tables. Merging code was also refactored
and now uses a SQL query to diff source and destination
tables in order to determine what to update or add.

Also fixes a bug where incremental imports involving
Excel files with multiple sheets would fail due to the UI
not serializing merge options correctly.

Test Plan: Browser tests.

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D3046
This commit is contained in:
George Gevoian
2021-10-04 09:14:14 -07:00
parent 7e07f0ce56
commit e1780e4f58
11 changed files with 473 additions and 885 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
app/client/components/Importer.ts
View File

@@ -18,6 +18,7 @@ import {icon} from 'app/client/ui2018/icons';
import {IOptionFull, linkSelect, multiSelect} from 'app/client/ui2018/menus';
import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals';
import {DataSourceTransformed, ImportResult, ImportTableResult, MergeOptions,
MergeOptionsMap,
MergeStrategy, TransformColumn, TransformRule, TransformRuleMap} from "app/common/ActiveDocAPI";
import {byteString} from "app/common/gutil";
import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
@@ -236,20 +237,8 @@ export class Importer extends Disposable {
return {uploadId: upload.uploadId, transforms};
}
private _getMergeOptions(upload: UploadResult): Array<MergeOptions|null> {
return upload.files.map((_file, i) => {
const sourceInfo = this._sourceInfoArray.get().find(info => info.uploadFileIndex === i);
if (!sourceInfo) { return null; }
const mergeOptions = this._mergeOptions[sourceInfo.hiddenTableId];
if (!mergeOptions) { return null; }
const {updateExistingRecords, mergeCols, mergeStrategy} = mergeOptions;
return {
mergeCols: updateExistingRecords.get() ? mergeCols.get() : [],
mergeStrategy: mergeStrategy.get()
};
});
private _getMergeOptionMaps(upload: UploadResult): MergeOptionsMap[] {
return upload.files.map((_file, i) => this._createMergeOptionsMap(i));
}
private _createTransformRuleMap(uploadFileIndex: number): TransformRuleMap {
@@ -262,6 +251,16 @@ export class Importer extends Disposable {
return result;
}
private _createMergeOptionsMap(uploadFileIndex: number): MergeOptionsMap {
const result: MergeOptionsMap = {};
for (const sourceInfo of this._sourceInfoArray.get()) {
if (sourceInfo.uploadFileIndex === uploadFileIndex) {
result[sourceInfo.origTableName] = this._getMergeOptionsForSource(sourceInfo);
}
}
return result;
}
private _createTransformRule(sourceInfo: SourceInfo): TransformRule {
const transformFields = sourceInfo.transformSection.get().viewFields().peek();
const sourceFields = sourceInfo.sourceSection.viewFields().peek();
@@ -279,6 +278,17 @@ export class Importer extends Disposable {
};
}
private _getMergeOptionsForSource(sourceInfo: SourceInfo): MergeOptions|undefined {
const mergeOptions = this._mergeOptions[sourceInfo.hiddenTableId];
if (!mergeOptions) { return undefined; }
const {updateExistingRecords, mergeCols, mergeStrategy} = mergeOptions;
return {
mergeCols: updateExistingRecords.get() ? mergeCols.get() : [],
mergeStrategy: mergeStrategy.get()
};
}
private _getHiddenTableIds(): string[] {
return this._sourceInfoArray.get().map((t: SourceInfo) => t.hiddenTableId);
}
@@ -332,10 +342,10 @@ export class Importer extends Disposable {
this._screen.renderSpinner();
const parseOptions = {...this._parseOptions.get(), NUM_ROWS: 0};
const mergeOptions = this._getMergeOptions(upload);
const mergeOptionMaps = this._getMergeOptionMaps(upload);
const importResult: ImportResult = await this._docComm.finishImportFiles(
this._getTransformedDataSource(upload), this._getHiddenTableIds(), {mergeOptions, parseOptions});
this._getTransformedDataSource(upload), this._getHiddenTableIds(), {mergeOptionMaps, parseOptions});
if (importResult.tables[0].hiddenTableId) {
const tableRowModel = this._gristDoc.docModel.dataTables[importResult.tables[0].hiddenTableId].tableMetaRow;