/** * Test of the Importer dialog (part 1), for imports inside an open doc. * (See Import.ts for tests from the DocMenu page.) */ import {assert, driver, Key} from 'mocha-webdriver'; import * as gu from 'test/nbrowser/gristUtils'; import {getColumnMatchingRows, getParseOptionInput, getPreviewDiffCellValues, openTableMapping, waitForColumnMapping, waitForDiffPreviewToLoad} from 'test/nbrowser/importerTestUtils'; import {setupTestSuite} from 'test/nbrowser/testUtils'; describe('Importer', function() { this.timeout(70000); // Imports can take some time, especially in tests that import larger files. const cleanup = setupTestSuite(); let docUrl: string|undefined; beforeEach(async function() { // Log in and import a sample document. If this is already done, we can skip these tests, to // have tests go faster. Each successful test case should leave the document unchanged. if (!docUrl || !await gu.testCurrentUrl(docUrl)) { const session = await gu.session().teamSite.login(); await session.tempDoc(cleanup, 'Hello.grist'); docUrl = await driver.getCurrentUrl(); } }); afterEach(() => gu.checkForErrors()); it('should show correct preview', async function() { await gu.importFileDialog('./uploads/UploadedData1.csv'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); assert.lengthOf(await driver.findAll('.test-importer-source'), 1); assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor' ]); // Check that the preview table cannot be edited by double-clicking a cell or via keyboard. const cell = await (await gu.getPreviewCell(0, 1)).doClick(); await driver.withActions(a => a.doubleClick(cell)); assert(await driver.find(".default_editor.readonly_editor").isPresent()); await gu.sendKeys(Key.ESCAPE); assert.isFalse(await driver.find(".default_editor.readonly_editor").isPresent()); await gu.sendKeys(Key.DELETE); await gu.waitForServer(); assert.equal(await cell.getText(), 'Lily'); // Check that the column matching section is not shown for new tables. assert.isFalse(await driver.find('.test-importer-column-match-options').isPresent()); // Check that the preview table doesn't show formula icons in cells. assert.isFalse(await cell.find('.formula_field').isPresent()); // Check that we have "Import Options" link and click it. assert.equal(await driver.find('.test-importer-options-link').isPresent(), true); await driver.find('.test-importer-options-link').click(); // Check that initially we see a button "Close" (nothing to update) assert.equal(await driver.findWait('.test-parseopts-back', 500).getText(), 'Close'); assert.equal(await driver.find('.test-parseopts-update').isPresent(), false); // After a change to parse options, button should change to 'Update Preview' await getParseOptionInput(/Field separator/).doClear().sendKeys("|"); assert.equal(await driver.findWait('.test-parseopts-update', 500).getText(), 'Update preview'); assert.equal(await driver.find('.test-parseopts-back').isPresent(), false); // Changing the parse option back to initial state reverts the button back too. await getParseOptionInput(/Field separator/).doClear().sendKeys(","); assert.equal(await driver.findWait('.test-parseopts-back', 500).getText(), 'Close'); assert.equal(await driver.find('.test-parseopts-update').isPresent(), false); // ensure that option 'First row contains headers' is checked if headers were guessed let useHeaders = await getParseOptionInput(/First row/); assert.equal(await useHeaders.getAttribute('checked'), 'true'); // Uncheck the option and update the preview. await useHeaders.click(); assert.equal(await useHeaders.getAttribute('checked'), null); await driver.find('.test-parseopts-update').click(); await gu.waitForServer(); // Ensure that column names become the first row in preview data. assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3, 4]), [ 'Name', 'Phone', 'Title', 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor' ]); // Check the option again and update the preview. await driver.find('.test-importer-options-link').click(); useHeaders = await getParseOptionInput(/First row/); assert.equal(await useHeaders.getAttribute('checked'), null); await useHeaders.click(); await driver.find('.test-parseopts-update').click(); await gu.waitForServer(); // Ensure that column names are used as headers again. assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor' ]); // Right-click a column header, to ensure we don't get a JS error in this case. const colHeader = await driver.findContent('.test-importer-preview .column_name', /Name/); await driver.withActions(actions => actions.contextClick(colHeader)); await gu.checkForErrors(); // Change Field separator and update the preview. await driver.find('.test-importer-options-link').click(); await getParseOptionInput(/Field separator/).doClick().sendKeys("|"); assert.equal(await getParseOptionInput(/Field separator/).value(), "|"); assert.equal(await getParseOptionInput(/Line terminator/).value(), "\\n"); await driver.find('.test-parseopts-update').click(); await gu.waitForServer(); assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3]), [ 'Lily,Jones,director', 'Kathy,Mills,student', 'Karen,Gold,professor' ]); // Close the dialog. await driver.find('.test-modal-cancel').click(); await gu.waitForServer(); assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); // No new pages should be present. assert.deepEqual(await gu.getPageNames(), ['Table1']); }); it('should show correct preview for multiple tables', async function() { await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv'); assert.equal(await driver.findWait('.test-importer-preview', 8000).isPresent(), true); assert.lengthOf(await driver.findAll('.test-importer-source'), 2); assert.equal(await driver.find('.test-importer-source-selected .test-importer-from').getText(), 'UploadedData1.csv'); assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor' ]); // Select another table await driver.findContent('.test-importer-from', /UploadedData2/).click(); await gu.waitForServer(); assert.equal(await driver.find('.test-importer-source-selected .test-importer-from').getText(), 'UploadedData2.csv'); assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', 'BUS540', 'Capstone', '', '01/13/2021', '' ]); // Check that changing a parse option (Field Separator to "|") affects both tables. await driver.find('.test-importer-options-link').click(); await getParseOptionInput(/Field separator/).doClick().sendKeys("|"); await driver.find('.test-parseopts-update').click(); await gu.waitForServer(); assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3]), [ 'Lily,Jones,director', 'Kathy,Mills,student', 'Karen,Gold,professor' ]); await driver.findContent('.test-importer-from', /UploadedData2/).click(); await gu.waitForServer(); assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3, 4, 5, 6]), [ 'BUS100,Intro to Business,,01/13/2021,false', 'BUS102,Business Law,Nathalie Patricia,01/13/2021,false', 'BUS300,Business Operations,Michael Rian,01/14/2021,false', 'BUS301,History of Business,Mariyam Melania,01/14/2021,false', 'BUS500,Ethics and Law,Filip Andries,01/13/2021,false', 'BUS540,Capstone,,01/13/2021,true' ]); // Close the dialog. await driver.find('.test-modal-cancel').click(); await gu.waitForServer(); assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); }); it('should not show preview for single empty file', async function() { await gu.importFileDialog('./uploads/UploadedDataEmpty.csv'); assert.match(await driver.findWait('.test-importer-error', 1000).getText(), /Import failed: No data was imported/); await driver.find('.test-modal-cancel').click(); await gu.waitForServer(); }); it('should not show preview for empty file when importing with non empty files', async function() { await gu.importFileDialog( './uploads/UploadedData1.csv,./uploads/UploadedData2.csv,./uploads/UploadedDataEmpty.csv'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); // Ensure that there are no empty tables shown. assert.deepEqual(await driver.findAll('.test-importer-from', (el) => el.getText()), ['UploadedData1.csv', 'UploadedData2.csv']); assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor' ]); await driver.findContent('.test-importer-from', /UploadedData2/).click(); await gu.waitForServer(); assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', 'BUS540', 'Capstone', '', '01/13/2021', '' ]); await driver.find('.test-modal-cancel').click(); await gu.waitForServer(); }); it('should finish import into an existing table', async function() { // First import the file into a new table, which is the default import action. await gu.importFileDialog('./uploads/UploadedData1.csv'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4], cols: [0, 1, 2] }), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor', '', '', '']); assert.deepEqual(await gu.getPageNames(), ['Table1', 'UploadedData1']); // Now import the same file again, choosing the same table as the first time. await gu.importFileDialog('./uploads/UploadedData1.csv'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); await driver.findContent('.test-importer-target-existing-table', /UploadedData1/).click(); await gu.waitForServer(); // The preview content should be the same, since all columns match. assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor' ]); await waitForColumnMapping(); assert.deepEqual(await getColumnMatchingRows(), [ { destination: 'Name', source: 'Name' }, { destination: 'Phone', source: 'Phone' }, { destination: 'Title', source: 'Title' }, ]); // Complete this second import. await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5, 6, 7], cols: [0, 1, 2] }), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor', 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor', '', '', '']); assert.deepEqual(await gu.getPageNames(), ['Table1', 'UploadedData1']); // Undo the import await gu.undo(2); // Ensure that imported table is removed, and we are back to the original one. assert.deepEqual(await gu.getPageNames(), ['Table1']); assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1], cols: [0, 1, 2] }), [ 'hello', '', '']); }); it('should finish import multiple files', async function() { // Import two files together. await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv'); await driver.findWait('.test-modal-confirm', 2000).click(); await gu.waitForServer(); assert.deepEqual(await gu.getPageNames(), ['Table1', 'UploadedData1', 'UploadedData2']); assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2], rowNums: [1, 2, 3]}), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor' ]); await gu.getPageItem('UploadedData2').click(); await gu.waitForServer(); assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5, 6]}), [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', 'BUS540', 'Capstone', '', '01/13/2021', '' ]); // Undo and check that we are back to the original state. await gu.undo(); assert.deepEqual(await gu.getPageNames(), ['Table1']); assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1], cols: [0, 1, 2] }), [ 'hello', '', '']); }); it('should import empty dates', async function() { await gu.importFileDialog('./uploads/EmptyDate.csv'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); // Finish import and check that the dialog gets closed. await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: [0, 1]}), [ "Bob", "2018-01-01", "Alice", "", "Carol", "2017-01-01" ]); assert.deepEqual(await gu.getPageNames(), ['Table1', 'EmptyDate']); // Add a new column, with a formula to examine the first. await gu.openColumnMenu('Birthday', 'Insert column to the right'); await driver.find('.test-new-columns-menu-add-new').click(); await gu.waitForServer(); await driver.sendKeys(Key.ESCAPE); await gu.getCell({col: 2, rowNum: 1}).click(); await driver.sendKeys('=type($Birthday).__name__', Key.ENTER); await gu.waitForServer(); // Ensure that there is no ValueError in second row assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: [0, 1, 2]}), [ "Bob", "2018-01-01", "date", "Alice", "", "NoneType", "Carol", "2017-01-01", "date" ]); }); it('should finish import xlsx file', async function() { await gu.importFileDialog('./uploads/homicide_rates.xlsx'); assert.equal(await driver.findWait('.test-importer-preview', 5000).isPresent(), true); await driver.find('.test-modal-confirm').click(); await gu.waitForServer(5000); assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); // Look at a small subset of the imported table. assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: [0, 1, 2]}), [ 'Africa', 'Eastern Africa', 'Burundi', 'Africa', 'Eastern Africa', 'Burundi', 'Africa', 'Eastern Africa', 'Comoros']); }); it('should import correctly in prefork mode', async function() { await driver.get(`${docUrl}/m/fork`); await gu.waitForDocToLoad(); await gu.importFileDialog('./uploads/homicide_rates.xlsx'); assert.equal(await driver.findWait('.test-importer-preview', 5000).isPresent(), true); await driver.find('.test-modal-confirm').click(); await gu.waitForServer(5000); assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); // Look at a small subset of the imported table. assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: [0, 1, 2]}), [ 'Africa', 'Eastern Africa', 'Burundi', 'Africa', 'Eastern Africa', 'Burundi', 'Africa', 'Eastern Africa', 'Comoros']); await driver.get(`${docUrl}`); await gu.acceptAlert(); await gu.waitForDocToLoad(); }); it('should support importing into on-demand tables', async function() { // Mark EmptyDate as on-demand. await gu.getPageItem('EmptyDate').click(); await gu.waitForServer(); await gu.toggleSidePanel('right', 'open'); await driver.find('.test-config-data').click(); await driver.find('[data-test-id=ViewConfig_advanced').click(); await driver.find('[data-test-id=ViewConfig_onDemandBtn').click(); await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); await gu.waitForDocToLoad(); // Import EmptyDate.csv into EmptyDate and check the import was successful. await gu.importFileDialog('./uploads/EmptyDate.csv'); assert.equal(await driver.findWait('.test-importer-preview', 5000).isPresent(), true); await driver.findContent('.test-importer-target-existing-table', /EmptyDate/).click(); await gu.waitForServer(); await driver.find('.test-modal-confirm').click(); await gu.waitForServer(5000); assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); // Check that the imported file contents were added to the end of EmptyDate. assert.deepEqual(await gu.getVisibleGridCells({rowNums: [4, 5, 6], cols: [0, 1]}), [ "Bob", "2018-01-01", "Alice", "", "Carol", "2017-01-01" ]); assert.equal(await gu.getGridRowCount(), 7); }); describe('when updating existing records', async function() { it('should populate merge columns/fields menu with columns from preview', async function() { // First import a file into a new table, so that we have a base for merging. await gu.importFileDialog('./uploads/UploadedData1.csv'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); // Now import the same file again. await gu.importFileDialog('./uploads/UploadedData1.csv'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); // Check that the 'Update existing records' checkbox is not visible (since destination is 'New Table'). assert.isNotTrue(await driver.find('.test-importer-update-existing-records').isPresent()); assert.isNotTrue(await driver.find('.test-importer-merge-fields-select').isPresent()); assert.isNotTrue(await driver.find('.test-importer-merge-fields-message').isPresent()); // Change the destination to the table we created earlier ('UploadedData1'). await driver.findContent('.test-importer-target-existing-table', /UploadedData1/).click(); await gu.waitForServer(); // Check that the 'Update existing records' checkbox is now visible and unchecked. assert(await driver.find('.test-importer-update-existing-records').isPresent()); assert.isNotTrue(await driver.find('.test-importer-merge-fields-select').isPresent()); assert.isNotTrue(await driver.find('.test-importer-merge-fields-message').isPresent()); // Click 'Update existing records' and verify that additional merge options are shown. await waitForColumnMapping(); await driver.find('.test-importer-update-existing-records').click(); assert.equal( await driver.find('.test-importer-merge-fields-message').getText(), 'Merge rows that match these fields:' ); assert.equal( await driver.find('.test-importer-merge-fields-select').getText(), 'Select fields to match on' ); // Open the field select menu and check that all the preview table columns are available options. await driver.find('.test-importer-merge-fields-select').click(); assert.deepEqual( await driver.findAll('.test-multi-select-menu .test-multi-select-menu-option-text', e => e.getText()), ['Name', 'Phone', 'Title'] ); // Close the field select menu. await gu.sendKeys(Key.ESCAPE); }); it('should display an error when clicking Import with no merge fields selected', async function() { // No merge fields are currently selected. Click Import and check that nothing happened. await driver.find('.test-modal-confirm').click(); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); // Check that the merge field select button has a red outline. assert.equal( await driver.find('.test-importer-merge-fields-select').getCssValue('border'), '1px solid rgb(208, 2, 27)' ); // Select a merge field, and check that the red outline is gone. await driver.find('.test-importer-merge-fields-select').click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Name/ ).click(); assert.equal( await driver.find('.test-importer-merge-fields-select').getCssValue('border'), '1px solid rgb(217, 217, 217)' ); // Hide dropdown await gu.sendKeys(Key.ESCAPE); await gu.checkForErrors(); }); it('should not throw an error when a column in the preview is clicked', async function() { // A bug was previosuly causing an error to be thrown whenever a column header was // clicked while merge columns were set. await driver.findContent('.test-importer-preview .column_name', /Name/).click(); await gu.checkForErrors(); }); it('should merge fields of matching records when Import is clicked', async function() { // The 'Name' field is selected as the only merge field. Click Import. assert.equal(await driver.find('.test-importer-merge-fields-select').getText(), 'Name'); await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); // Check that the destination table is unchanged since we imported the same file. assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4], cols: [0, 1, 2] }), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor', '', '', '' ] ); // Undo the import, and check that the destination table is still unchanged. await gu.undo(); assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4], cols: [0, 1, 2] }), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor', '', '', '' ] ); // Import from another file containing some duplicates (with new values). await gu.importFileDialog('./uploads/UploadedData1Extended.csv'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); await driver.findContent('.test-importer-target-existing-table', /UploadedData1/).click(); await gu.waitForServer(); // Set the merge fields to 'Name' and 'Phone'. await waitForColumnMapping(); await driver.find('.test-importer-update-existing-records').click(); await driver.find('.test-importer-merge-fields-select').click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Name/ ).click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Phone/ ).click(); // Close the merge fields menu. await gu.sendKeys(Key.ESCAPE); assert.equal(await driver.find('.test-importer-merge-fields-select').getText(), 'Name, Phone'); // Check the preview shows a diff of the changes importing will make. await waitForDiffPreviewToLoad(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2], [1, 2, 3, 4, 5, 6]), [ 'Lily', 'Jones', ['director', 'student', undefined], 'Kathy', 'Mills', ['student', 'professor', undefined], 'Karen', 'Gold', ['professor', 'director', undefined], [undefined, 'Michael', undefined], [undefined, 'Smith', undefined], [undefined, 'student', undefined], [undefined, 'Lily', undefined], [undefined, 'James', undefined], [undefined, 'student', undefined], '', '', '', ] ); // Complete the import, and verify that incoming data was merged into matching records in UploadedData1. await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5, 6], cols: [0, 1, 2] }), [ 'Lily', 'Jones', 'student', 'Kathy', 'Mills', 'professor', 'Karen', 'Gold', 'director', 'Michael', 'Smith', 'student', 'Lily', 'James', 'student', '', '', '' ] ); // Undo the import, and check the table is back to how it was pre-import. await gu.undo(); assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4], cols: [0, 1, 2] }), [ 'Lily', 'Jones', 'director', 'Kathy', 'Mills', 'student', 'Karen', 'Gold', 'professor', '', '', '' ] ); }); it('should support merging multiple CSV files into multiple tables', async function() { // Import a second table, so we have 2 destinations to incrementally import into. await gu.importFileDialog('./uploads/UploadedData2.csv'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); // Now import new versions of both files together. await gu.importFileDialog('./uploads/UploadedData1Extended.csv,./uploads/UploadedData2Extended.csv'); // For UploadedData1Extended.csv, check 'Update existing records', but don't pick any merge fields yet. await driver.findContent('.test-importer-target-existing-table', /UploadedData1/).click(); await gu.waitForServer(); await waitForColumnMapping(); await driver.find('.test-importer-update-existing-records').click(); // Try to click on UploadedData2.csv. await driver.findContent('.test-importer-source', /UploadedData2Extended.csv/).click(); // Check that it failed, and that the merge fields select button is outlined in red. assert.equal( await driver.find('.test-importer-merge-fields-select').getCssValue('border'), '1px solid rgb(208, 2, 27)' ); assert.equal( await driver.find('.test-importer-source-selected .test-importer-from').getText(), 'UploadedData1Extended.csv' ); // Now pick the merge fields, and check that the preview diff looks correct. await driver.find('.test-importer-merge-fields-select').click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Name/ ).click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Phone/ ).click(); await gu.sendKeys(Key.ESCAPE); await waitForDiffPreviewToLoad(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2], [1, 2, 3, 4, 5, 6]), [ 'Lily', 'Jones', ['director', 'student', undefined], 'Kathy', 'Mills', ['student', 'professor', undefined], 'Karen', 'Gold', ['professor', 'director', undefined], [undefined, 'Michael', undefined], [undefined, 'Smith', undefined], [undefined, 'student', undefined], [undefined, 'Lily', undefined], [undefined, 'James', undefined], [undefined, 'student', undefined], '', '', '', ] ); // Check that clicking UploadedData2 now works. await driver.findContent('.test-importer-source', /UploadedData2Extended.csv/).click(); await gu.waitForServer(); await driver.findContent('.test-importer-target-existing-table', /UploadedData2/).click(); await gu.waitForServer(); assert.equal( await driver.find('.test-importer-source-selected .test-importer-from').getText(), 'UploadedData2Extended.csv' ); // Set the merge fields for UploadedData2 to 'CourseId'. await waitForColumnMapping(); await driver.find('.test-importer-update-existing-records').click(); await driver.find('.test-importer-merge-fields-select').click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /CourseId/ ).click(); // Close the merge fields menu. await gu.sendKeys(Key.ESCAPE); assert.equal(await driver.find('.test-importer-merge-fields-select').getText(), 'CourseId'); // Check that the preview diff looks correct for UploadedData2. await waitForDiffPreviewToLoad(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7, 8, 9]), [ 'BUS100', 'Intro to Business', [undefined, 'Mariyam Melania', undefined], '01/13/2021', '', 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', 'BUS500', [undefined, undefined, 'Ethics and Law'], 'Filip Andries', '01/13/2021', '', [undefined, 'BUS501', undefined], [undefined, 'Marketing', undefined], [undefined, 'Michael Rian', undefined], [undefined, '01/13/2021', undefined], [undefined, 'false', undefined], [undefined, 'BUS539', undefined], [undefined, 'Independent Study', undefined], '', [undefined, '01/13/2021', undefined], [undefined, 'true', undefined], 'BUS540', 'Capstone', '', '01/13/2021', ['true', 'false', undefined], '', '', '', '', '' ] ); // Complete the import, and verify that incoming data was merged into both UploadedData1 and UploadedData2. await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); assert.deepEqual(await gu.getPageNames(), [ 'Table1', 'EmptyDate', 'Homicide counts and rates (2000', 'Sheet1', 'UploadedData1', 'UploadedData2' ]); // Check the contents of UploadedData1. assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5, 6], cols: [0, 1, 2] }), [ 'Lily', 'Jones', 'student', 'Kathy', 'Mills', 'professor', 'Karen', 'Gold', 'director', 'Michael', 'Smith', 'student', 'Lily', 'James', 'student', '', '', '' ] ); // Check the contents of UploadedData2. await gu.getPageItem('UploadedData2').click(); await gu.waitForServer(); assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5, 6, 7, 8, 9]}), [ 'BUS100', 'Intro to Business', 'Mariyam Melania', '01/13/2021', '', 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', 'BUS540', 'Capstone', '', '01/13/2021', '', 'BUS501', 'Marketing', 'Michael Rian', '01/13/2021', '', 'BUS539', 'Independent Study', '', '01/13/2021', '', '', '', '', '', '' ]); }); it('should support merging multiple Excel sheets into multiple tables', async function() { this.timeout(90000); // Import an Excel file with multiple sheets into new tables. await gu.importFileDialog('./uploads/World-v0.xlsx'); assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); // Now import a new version of the Excel file with updated data. await gu.importFileDialog('./uploads/World-v1.xlsx'); // For sheet Table1, don't pick any merge fields and import into the existing table (Table1_2). await driver.findContent('.test-importer-target-existing-table', /Table1_2/).click(); await gu.waitForServer(); // For sheet City, merge on Name, District and Country. await driver.findContent('.test-importer-source', /City/).click(); await gu.waitForServer(); await driver.findContent('.test-importer-target-existing-table', /City/).click(); await gu.waitForServer(); await waitForColumnMapping(); await driver.find('.test-importer-update-existing-records').click(); await driver.find('.test-importer-merge-fields-select').click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Name/ ).click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /District/ ).click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Country/ ).click(); await gu.sendKeys(Key.ESCAPE); // Check the preview diff of City. The population should have doubled in every row. await waitForDiffPreviewToLoad(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5]), [ 'Kabul', 'Kabol', ['1780000', '3560000', undefined], '2', ['1780', '3560', undefined], 'Qandahar', 'Qandahar', ['237500', '475000', undefined], '2', ['237.5', '475', undefined], 'Herat', 'Herat', ['186800', '373600', undefined], '2', ['186.8', '373.6', undefined], 'Mazar-e-Sharif', 'Balkh', ['127800', '255600', undefined], '2', ['127.8', '255.6', undefined], 'Amsterdam', 'Noord-Holland', ['731200', '1462400', undefined], '159', ['731.2', '1462.4', undefined], ] ); // For sheet Country, merge on Code. await driver.findContent('.test-importer-source', /Country/).click(); await gu.waitForServer(); await driver.findContent('.test-importer-target-existing-table', /Country/).click(); await gu.waitForServer(); await waitForColumnMapping(); await driver.find('.test-importer-update-existing-records').click(); await driver.find('.test-importer-merge-fields-select').click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Code/ ).click(); await gu.sendKeys(Key.ESCAPE); // Check the preview diff of Country. The population should have doubled in every row. await waitForDiffPreviewToLoad(); assert.deepEqual( await getPreviewDiffCellValues([0, 6], [1, 2, 3, 4, 5]), [ 'ABW', ['103000', '206000', undefined], 'AFG', ['22720000', '45440000', undefined], 'AGO', ['12878000', '25756000', undefined], 'AIA', ['8000', '16000', undefined], 'ALB', [ '3401200', '6802400', undefined] ] ); // For sheet CountryLanguage, merge on Country and Language. await driver.findContent('.test-importer-source', /CountryLanguage/).click(); await gu.waitForServer(); await driver.findContent('.test-importer-target-existing-table', /CountryLanguage/).click(); await gu.waitForServer(); await waitForColumnMapping(); await driver.find('.test-importer-update-existing-records').click(); await driver.find('.test-importer-merge-fields-select').click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Country/ ).click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /Language/ ).click(); await gu.sendKeys(Key.ESCAPE); // Check the preview diff of CountryLanguage. The first few percentages should be slightly different. await waitForDiffPreviewToLoad(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3], [1, 2, 3, 4, 5]), [ 'Dutch', ['5.3', '5.5', undefined], 'ABW', '', 'English', ['9.5', '9.3', undefined], 'ABW', '', 'Papiamento', ['76.7', '76.3', undefined], 'ABW', '', 'Spanish', ['7.4', '7.8', undefined], 'ABW', '', 'Balochi', ['0.9', '1.1', undefined], 'AFG', '' ] ); // Complete the import, and verify that incoming data was merged correctly. await driver.find('.test-modal-confirm').click(); await gu.waitForServer(); assert.deepEqual(await gu.getPageNames(), [ 'Table1', 'EmptyDate', 'Homicide counts and rates (2000', 'Sheet1', 'UploadedData1', 'UploadedData2', 'Table1', 'City', 'Country', 'CountryLanguage' ]); // Check the contents of Table1; it should have duplicates of the original 2 rows. assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5], cols: [0, 1, 2, 3, 4] }), [ 'hello', '', '', '', 'HELLO', '', 'world', '', '', '', 'hello', '', '', '', 'HELLO', '', 'world', '', '', '', '', '', '', '', '', ] ); // Check the contents of City. The population should have doubled in every row. await gu.getPageItem('City').click(); await gu.waitForServer(); assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5]}), [ 'Kabul', 'Kabol', '3560000', '2', '3560', 'Qandahar', 'Qandahar', '475000', '2', '475', 'Herat', 'Herat', '373600', '2', '373.6', 'Mazar-e-Sharif', 'Balkh', '255600', '2', '255.6', 'Amsterdam', 'Noord-Holland', '1462400', '159', '1462.4', ] ); // Check that no new rows were added to City. assert.equal(await gu.getGridRowCount(), 4080); // Check the contents of Country. The population should have doubled in every row. await gu.getPageItem('Country').click(); await gu.waitForServer(); assert.deepEqual( await gu.getVisibleGridCells({ cols: [0, 6], rowNums: [1, 2, 3, 4, 5] }), [ 'ABW', '206000', 'AFG', '45440000', 'AGO', '25756000', 'AIA', '16000', 'ALB', '6802400' ] ); // Check that no new rows were added to Country. assert.equal(await gu.getGridRowCount(), 240); // Check the contents of CountryLanguage. The first few percentages should be slightly different. await gu.getPageItem('CountryLanguage').click(); await gu.waitForServer(); assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3], rowNums: [1, 2, 3, 4, 5]}), [ 'Dutch', '5.5', 'ABW', '', 'English', '9.3', 'ABW', '', 'Papiamento', '76.3', 'ABW', '', 'Spanish', '7.8', 'ABW', '', 'Balochi', '1.1', 'AFG', '' ] ); // Check that no new rows were added to CountryLanguage. assert.equal(await gu.getGridRowCount(), 985); }); it('should show diff of changes in preview', async function() { // Import UploadedData2.csv again, and change the destination to UploadedData2. await gu.importFileDialog('./uploads/UploadedData2.csv'); await driver.findContent('.test-importer-target-existing-table', /UploadedData2/).click(); await gu.waitForServer(); await waitForColumnMapping(); // Click 'Update existing records', and check the preview does not yet show a diff. await driver.find('.test-importer-update-existing-records').click(); await gu.waitForServer(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', 'BUS540', 'Capstone', '', '01/13/2021', '', '', '', '', '', '' ]); // Select 'CourseId' as the merge column, and check that the preview now contains a diff of old/new values. await driver.find('.test-importer-merge-fields-select').click(); await driver.findContent( '.test-multi-select-menu .test-multi-select-menu-option', /CourseId/ ).click(); await gu.sendKeys(Key.ESCAPE); await gu.waitForServer(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), [ 'BUS100', 'Intro to Business', [undefined, undefined, 'Mariyam Melania'], '01/13/2021', '', 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', 'BUS540', 'Capstone', '', '01/13/2021', ['false', 'true', undefined], '', '', '', '', '' ]); // Uncheck 'Update existing records', and check that the preview no longer shows a diff. await driver.find('.test-importer-update-existing-records').click(); await waitForDiffPreviewToLoad(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', 'BUS540', 'Capstone', '', '01/13/2021', '', '', '', '', '', '' ]); // Check that the column matching section is correct. assert.deepEqual(await getColumnMatchingRows(), [ { destination: 'CourseId', source: 'CourseId' }, { destination: 'CourseName', source: 'CourseName' }, { destination: 'Instructor', source: 'Instructor' }, { destination: 'StartDate', source: 'StartDate' }, { destination: 'PassFail', source: 'PassFail' }, ]); // Click 'Update existing records' again, and edit the formula for CourseId to append a suffix. await driver.find('.test-importer-update-existing-records').click(); await waitForDiffPreviewToLoad(); await driver.findContent('.test-importer-column-match-source-destination', /CourseId/) .find('.test-importer-column-match-formula').click(); await driver.find('.test-importer-apply-formula').click(); await gu.sendKeys(' + "-NEW"'); // Before saving the formula, check that the preview isn't showing the hidden helper column ids. assert.deepEqual( await driver.find('.test-importer-preview').findAll('.g-column-label', el => el.getText()), ['CourseId', 'CourseName', 'Instructor', 'StartDate', 'PassFail'] ); await gu.sendKeys(Key.ENTER); await gu.waitForServer(); // Check that the preview diff was updated and now shows that all 6 rows are new rows. await waitForDiffPreviewToLoad(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), [ [undefined, 'BUS100-NEW', undefined], [undefined, 'Intro to Business', undefined], '', [undefined, '01/13/2021', undefined], [undefined, 'false', undefined], [undefined, 'BUS102-NEW', undefined], [undefined, 'Business Law', undefined], [undefined, 'Nathalie Patricia', undefined], [undefined, '01/13/2021', undefined], [undefined, 'false', undefined], [undefined, 'BUS300-NEW', undefined], [undefined, 'Business Operations', undefined], [undefined, 'Michael Rian', undefined], [undefined, '01/14/2021', undefined], [undefined, 'false', undefined], [undefined, 'BUS301-NEW', undefined], [undefined, 'History of Business', undefined], [undefined, 'Mariyam Melania', undefined], [undefined, '01/14/2021', undefined], [undefined, 'false', undefined], [undefined, 'BUS500-NEW', undefined], [undefined, 'Ethics and Law', undefined], [undefined, 'Filip Andries', undefined], [undefined, '01/13/2021', undefined], [undefined, 'false', undefined], [undefined, 'BUS540-NEW', undefined], [undefined, 'Capstone', undefined], '', [undefined, '01/13/2021', undefined], [undefined, 'true', undefined], '', '', '', '', '' ] ); // Check the column mapping section updated with the new formula. assert.deepEqual(await getColumnMatchingRows(), [ { destination: 'CourseId', source: '$CourseId + "-NEW"\n' }, { destination: 'CourseName', source: 'CourseName' }, { destination: 'Instructor', source: 'Instructor' }, { destination: 'StartDate', source: 'StartDate' }, { destination: 'PassFail', source: 'PassFail' }, ]); // Change the destination back to new table, and check that the preview no longer shows a diff. await openTableMapping(); await driver.find('.test-importer-target-new-table').click(); await gu.waitForServer(); assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', 'BUS540', 'Capstone', '', '01/13/2021', '', '', '', '', '', '' ]); // Close the dialog. await driver.find('.test-modal-cancel').click(); await gu.waitForServer(); }); }); });