(core) Adding backend for 2-way references

Summary: Adding support for 2-way references in data engine. - Columns have an `reverseCol` field, which says "this is a reverse of the given column, update me when that one changes". - At the time of setting `reverseCol`, we ensure that it's symmetrical to make a 2-way reference. - Elsewhere we just implement syncing in one direction: - When `reverseCol` is present, user code is generated with a type like `grist.ReferenceList("Tasks", reverse_of="Assignee")` - On updating a ref column, we use `prepare_new_values()` method to generate corresponding updates to any column that's a reverse of it. - The `prepare_new_values()` approach is extended to support this. - We don't add (or remove) any mappings between rows, and rely on existing mappings (in a ref column's `_relation`) to create reverse updates. NOTE This is polished version of https://phab.getgrist.com/D4307 with tests and 3 bug fixes - Column transformation didn't work when transforming RefList to Ref, the reverse column became out of sync - Tables with reverse columns couldn't be removed - Setting json arrays to RefList didn't work if arrays contained other things besides ints Those fixes are covered by new tests. Test Plan: New tests Reviewers: georgegevoian, paulfitz, dsagal Reviewed By: georgegevoian, paulfitz Subscribers: dsagal Differential Revision: https://phab.getgrist.com/D4322
2026-03-02 04:09:24 +00:00 · 2024-09-09 18:24:11 +02:00
parent 0ca70e9d43
commit 1d2cf3de49
24 changed files with 2164 additions and 115 deletions
--- a/test/nbrowser/DropdownConditionEditor.ts
+++ b/test/nbrowser/DropdownConditionEditor.ts
@@ -66,7 +66,7 @@ describe('DropdownConditionEditor', function () {
          'user.A\nc\ncess\n ',
        ]);
      });
-      await gu.sendKeysSlowly(['hoice not in ']);
+      await gu.sendKeysSlowly('hoice not in ');
      // Attempts to reduce test flakiness by delaying input of $. Not guaranteed to do anything.
      await driver.sleep(100);
      await gu.sendKeys('$');
@@ -192,7 +192,7 @@ describe('DropdownConditionEditor', function () {
      assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent());
      await driver.find('.test-field-set-dropdown-condition').click();
      await gu.waitAppFocus(false);
-      await gu.sendKeysSlowly(['choice']);
+      await gu.sendKeysSlowly('choice');
      await gu.waitToPass(async () => {
        const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText());
        assert.deepEqual(completions, [
@@ -340,7 +340,7 @@ describe('DropdownConditionEditor', function () {
    await gu.getCell(1, 1).click();
    await driver.find('.test-field-set-dropdown-condition').click();
    await gu.waitAppFocus(false);
-    await gu.sendKeysSlowly(['user.']);
+    await gu.sendKeysSlowly('user.');
    await gu.waitToPass(async () => {
      const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText());
      assert.deepEqual(completions, [
--- a/test/nbrowser/Formulas.ts
+++ b/test/nbrowser/Formulas.ts
@@ -231,13 +231,13 @@ return [
    await driver.sendKeys('=');
    await gu.waitAppFocus(false);
    // A single long string often works, but sometimes fails, so break up into multiple.
-    await gu.sendKeys(`  if $Budget > 50:${Key.chord(Key.SHIFT, Key.ENTER)}`);
+    await gu.sendKeysSlowly(`  if $Budget > 50:${Key.chord(Key.SHIFT, Key.ENTER)}`);
    await driver.sleep(50);
    // The next line should get auto-indented.
-    await gu.sendKeys(`return 'Big'${Key.chord(Key.SHIFT, Key.ENTER)}`);
+    await gu.sendKeysSlowly(`return 'Big'${Key.chord(Key.SHIFT, Key.ENTER)}`);
    await driver.sleep(50);
    // In the next line, we want to remove one level of indent.
-    await gu.sendKeys(`${Key.BACK_SPACE}return 'Small'`);
+    await gu.sendKeysSlowly(`${Key.BACK_SPACE}return 'Small'`);
    await gu.sendKeys(Key.ENTER);
    await gu.waitForServer();

--- a/test/nbrowser/TwoWayReference.ts
+++ b/test/nbrowser/TwoWayReference.ts
@@ -0,0 +1,382 @@
+import {assert, driver, Key} from 'mocha-webdriver';
+import * as gu from 'test/nbrowser/gristUtils';
+import {Session} from 'test/nbrowser/gristUtils';
+import {setupTestSuite} from 'test/nbrowser/testUtils';
+
+describe('TwoWayReference', function() {
+  this.timeout('3m');
+  let session: Session;
+  let docId: string;
+  const cleanup = setupTestSuite();
+  afterEach(() => gu.checkForErrors());
+  before(async function() {
+    session = await gu.session().login();
+    docId = await session.tempNewDoc(cleanup);
+    await gu.toggleSidePanel('left', 'close');
+    await petsSetup();
+  });
+
+  async function petsSetup() {
+    await gu.sendActions([
+      ['RenameColumn', 'Table1', 'A', 'Name'],
+      ['ModifyColumn', 'Table1', 'Name', {label: 'Name'}],
+      ['RemoveColumn', 'Table1', 'B'],
+      ['RemoveColumn', 'Table1', 'C'],
+      ['RenameTable', 'Table1', 'Owners'],
+      ['AddTable', 'Pets', [
+        {id: 'Name', type: 'Text'},
+        {id: 'Owner', type: 'Ref:Owners'},
+      ]],
+      ['AddRecord', 'Owners', -1, {Name: 'Alice'}],
+      ['AddRecord', 'Owners', -2, {Name: 'Bob'}],
+      ['AddRecord', 'Pets', null, {Name: 'Rex', Owner: -2}],
+    ]);
+    await gu.addNewSection('Table', 'Pets');
+    await gu.openColumnPanel('Owner');
+    await gu.setRefShowColumn('Name');
+    await addReverseColumn('Pets', 'Owner');
+  }
+
+  it('deletes tables with 2 way references', async function() {
+    const revert = await gu.begin();
+    await gu.toggleSidePanel('left', 'open');
+    await driver.find('.test-tools-raw').click();
+    const removeTable = async (tableId: string) => {
+      await driver.findWait(`.test-raw-data-table-menu-${tableId}`, 1000).click();
+      await driver.find('.test-raw-data-menu-remove-table').click();
+      await driver.find('.test-modal-confirm').click();
+      await gu.waitForServer();
+    };
+    await removeTable('Pets');
+    await revert();
+    await removeTable('Owners');
+    await gu.checkForErrors();
+    await revert();
+    await gu.openPage('Table1');
+  });
+
+  it('detects new columns after modify', async function() {
+    const revert = await gu.begin();
+
+    await gu.selectSectionByTitle('Owners');
+    await gu.selectColumn('Pets');
+    await gu.setType('Reference', {apply: true});
+    await gu.setType('Reference List', {apply: true});
+
+    await gu.selectSectionByTitle('Pets');
+    await gu.getCell('Owner', 1).click();
+    await gu.sendKeys(Key.DELETE);
+    await gu.waitForServer();
+
+    await gu.selectSectionByTitle('Owners');
+    assert.deepEqual(await gu.getVisibleGridCells('Pets', [1, 2]), ['', '']);
+    await revert();
+  });
+
+  it('can delete reverse column without an error', async function() {
+    const revert = await gu.begin();
+    // This can't be tested easily in python as it requries node server for type transformation.
+    await gu.toggleSidePanel('left', 'close');
+    await gu.toggleSidePanel('right', 'close');
+
+    // Remove the reverse column.
+    await gu.selectSectionByTitle('OWNERS');
+    await gu.deleteColumn('Pets');
+    await gu.checkForErrors();
+
+    // Check data.
+    assert.deepEqual(await columns(), [
+      ['Name'],
+      ['Name', 'Owner']
+    ]);
+    assert.deepEqual(await gu.getVisibleGridCells('Owner', [1], 'PETS'), ['Bob']);
+    await gu.undo();
+
+    // Check data.
+    assert.deepEqual(await columns(), [
+      ['Name', 'Pets'],
+      ['Name', 'Owner']
+    ]);
+    assert.deepEqual(await gu.getVisibleGridCells('Pets', [1, 2], 'OWNERS'), ['', 'Rex']);
+    assert.deepEqual(await gu.getVisibleGridCells('Owner', [1], 'PETS'), ['Bob']);
+
+    // Check that connection works.
+
+    // Make sure we can change data.
+    await gu.selectSectionByTitle('PETS');
+    await gu.getCell('Owner', 1).click();
+    await gu.enterCell('Alice', Key.ENTER);
+    await gu.waitForServer();
+    await gu.checkForErrors();
+
+    // Check data.
+    assert.deepEqual(await gu.getVisibleGridCells('Owner', [1], 'PETS'), ['Alice']);
+    assert.deepEqual(await gu.getVisibleGridCells('Pets', [1, 2], 'OWNERS'), ['Rex', '']);
+
+    // Now delete Owner column, and redo it
+    await gu.selectSectionByTitle('Pets');
+    await gu.deleteColumn('Owner');
+    await gu.checkForErrors();
+    await gu.undo();
+    await gu.checkForErrors();
+
+    // Check data.
+    assert.deepEqual(await gu.getVisibleGridCells('Owner', [1], 'PETS'), ['Alice']);
+    assert.deepEqual(await gu.getVisibleGridCells('Pets', [1, 2], 'OWNERS'), ['Rex', '']);
+    await revert();
+  });
+
+  it('breaks connection after removing reverseCol', async function() {
+    const revert = await gu.begin();
+
+    // Make sure Rex is owned by Bob, in both tables.
+    await gu.assertGridData('OWNERS', [
+      [0, "Name", "Pets"],
+      [1, "Alice", ""],
+      [2, "Bob",   "Rex"],
+    ]);
+    await gu.assertGridData("PETS", [
+      [0, "Name", "Owner"],
+      [1, "Rex",  "Bob"],
+    ]);
+
+    // Now move Rex to Alice.
+    await gu.selectSectionByTitle('PETS');
+    await gu.getCell('Owner', 1).click();
+    await gu.enterCell("Alice", Key.ENTER);
+    await gu.waitForServer();
+    await gu.assertGridData('OWNERS', [
+      [0, "Name", "Pets"],
+      [1, "Alice", "Rex"],
+      [2, "Bob", ""],
+    ]);
+
+    // Now remove connection using Owner column.
+    await gu.sendActions([['ModifyColumn', 'Pets', 'Owner', {reverseCol: 0}]]);
+    await gu.checkForErrors();
+
+    // And check that after moving Rex to Bob, it's not shown in the Owners table.
+    await gu.selectSectionByTitle('PETS');
+    await gu.getCell('Owner', 1).click();
+    await gu.enterCell("Bob", Key.ENTER);
+    await gu.waitForServer();
+    await gu.checkForErrors();
+
+    await gu.assertGridData('OWNERS', [
+      [0, "Name", "Pets"],
+      [1, "Alice", "Rex"],
+      [2, "Bob", ""],
+    ]);
+    await gu.assertGridData("PETS", [
+      [0, "Name", "Owner"],
+      [1, "Rex",  "Bob"],
+    ]);
+
+    // Check undo, it should restore the link.
+    await gu.undo(2);
+
+    // Rex is now in Alice again in both tables.
+    await gu.assertGridData('OWNERS', [
+      [0, "Name", "Pets"],
+      [1, "Alice", "Rex"],
+      [2, "Bob", ""],
+    ]);
+    await gu.assertGridData("PETS", [
+      [0, "Name", "Owner"],
+      [1, "Rex",  "Alice"],
+    ]);
+
+    // Move Rex to Bob again.
+    await gu.selectSectionByTitle('PETS');
+    await gu.getCell('Owner', 1).click();
+    await gu.enterCell("Bob", Key.ENTER);
+    await gu.waitForServer();
+    await gu.checkForErrors();
+
+    // And check that connection works.
+    await gu.assertGridData('OWNERS', [
+      [0, "Name", "Pets"],
+      [1, "Alice", ""],
+      [2, "Bob", "Rex"],
+    ]);
+    await gu.assertGridData("PETS", [
+      [0, "Name", "Owner"],
+      [1, "Rex",  "Bob"],
+    ]);
+    await revert();
+  });
+
+  it('works after reload', async function() {
+    const revert = await gu.begin();
+
+    await gu.selectSectionByTitle('OWNERS');
+    assert.deepEqual(await gu.getVisibleGridCells('Pets', [1, 2]), ['', 'Rex']);
+    await session.createHomeApi().getDocAPI(docId).forceReload();
+    await driver.navigate().refresh();
+    await gu.waitForDocToLoad();
+    // Change Rex owner to Alice.
+    await gu.selectSectionByTitle('PETS');
+    await gu.getCell('Owner', 1).click();
+    await gu.sendKeys('Alice', Key.ENTER);
+    await gu.waitForServer();
+    await gu.selectSectionByTitle('OWNERS');
+    assert.deepEqual(await gu.getVisibleGridCells('Pets', [1, 2]), ['Rex', '']);
+    await revert();
+  });
+
+  async function projectSetup() {
+    await gu.sendActions([
+      ['AddTable', 'Projects', []],
+      ['AddTable', 'People', []],
+
+      ['AddVisibleColumn', 'Projects', 'Name', {type: 'Text'}],
+      ['AddVisibleColumn', 'Projects', 'Owner', {type: 'Ref:People'}],
+
+      ['AddVisibleColumn', 'People', 'Name', {type: 'Text'}],
+    ]);
+    await gu.addNewPage('Table', 'Projects');
+    await gu.addNewSection('Table', 'People');
+    await gu.selectSectionByTitle('Projects');
+    await gu.openColumnPanel();
+    await gu.toggleSidePanel('left', 'close');
+  }
+
+  it('undo works for adding reverse column', async function() {
+    await projectSetup();
+    const revert = await gu.begin();
+
+    assert.deepEqual(await columns(), [
+      ['Name', 'Owner'],
+      ['Name']
+    ]);
+    await addReverseColumn('Projects', 'Owner');
+    assert.deepEqual(await columns(), [
+      ['Name', 'Owner'],
+      ['Name', 'Projects']
+    ]);
+    await gu.undo(1);
+    assert.deepEqual(await columns(), [
+      ['Name', 'Owner'],
+      ['Name']
+    ]);
+    await gu.redo(1);
+    assert.deepEqual(await columns(), [
+      ['Name', 'Owner'],
+      ['Name', 'Projects']
+    ]);
+    await revert();
+  });
+
+  it('creates proper names when added multiple times', async function() {
+    const revert = await gu.begin();
+    await addReverseColumn('Projects', 'Owner');
+
+    // Add another reference to Projects from People.
+    await gu.selectSectionByTitle('Projects');
+    await gu.addColumn('Tester', 'Reference');
+    await gu.setRefTable('People');
+    await gu.setRefShowColumn('Name');
+
+    // And now show it on People.
+    await addReverseColumn('Projects', 'Tester');
+
+    // We should now see 3 columns on People.
+    await gu.selectSectionByTitle('People');
+    assert.deepEqual(await gu.getColumnNames(), ['Name', 'Projects', 'Projects_Tester']);
+
+    // Add yet another one.
+    await gu.selectSectionByTitle('Projects');
+    await gu.addColumn('PM', 'Reference');
+    await gu.setRefTable('People');
+    await gu.setRefShowColumn('Name');
+    await addReverseColumn('Projects', 'PM');
+
+    // We should now see 4 columns on People.
+    await gu.selectSectionByTitle('People');
+    assert.deepEqual(await gu.getColumnNames(), ['Name', 'Projects', 'Projects_Tester', 'Projects_PM']);
+
+    await revert();
+  });
+
+  it('works well for self reference', async function() {
+    const revert = await gu.begin();
+
+    // Create a new table with task hierarchy and check if looks sane.
+    await gu.addNewPage('Table', 'New Table', {
+      tableName: 'Tasks',
+    });
+    await gu.renameColumn('A', 'Name');
+    await gu.renameColumn('B', 'Parent');
+    await gu.sendActions([
+      ['RemoveColumn', 'Tasks', 'C']
+    ]);
+    await gu.setType('Reference');
+    await gu.setRefTable('Tasks');
+    await gu.setRefShowColumn('Name');
+    await gu.sendActions([
+      ['AddRecord', 'Tasks', -1, {Name: 'Parent'}],
+      ['AddRecord', 'Tasks', null, {Name: 'Child', Parent: -1}],
+    ]);
+    await gu.openColumnPanel('Parent');
+    await addReverseColumn('Tasks', 'Parent');
+
+    // We should now see 3 columns on Tasks.
+    assert.deepEqual(await gu.getColumnNames(), ['Name', 'Parent', 'Tasks']);
+
+    await gu.openColumnPanel('Tasks');
+    await gu.setRefShowColumn('Name');
+
+    // Check that data looks ok.
+    assert.deepEqual(await gu.getVisibleGridCells('Name', [1, 2]), ['Parent', 'Child']);
+    assert.deepEqual(await gu.getVisibleGridCells('Parent', [1, 2]), ['', 'Parent']);
+    assert.deepEqual(await gu.getVisibleGridCells('Tasks', [1, 2]), ['Child', '']);
+
+    await revert();
+  });
+
+  it('converts from RefList to Ref without problems', async function() {
+    await session.tempNewDoc(cleanup);
+    const revert = await gu.begin();
+    await gu.sendActions([
+      ['AddTable', 'People', [
+        {id: 'Name', type: 'Text'},
+        {id: 'Supervisor', type: 'Ref:People'},
+      ]],
+      ['AddRecord', 'People', 1, {Name: 'Alice'}],
+      ['AddRecord', 'People', 4, {Name: 'Bob'}],
+      ['UpdateRecord', 'People', 1, {Supervisor: 4}],
+      ['UpdateRecord', 'People', 3, {Supervisor: 0}],
+    ]);
+
+    await gu.toggleSidePanel('left', 'open');
+    await gu.openPage('People');
+    await gu.openColumnPanel('Supervisor');
+    await gu.setRefShowColumn('Name');
+
+    // Using the convert dialog caused an error, which wasn't raised when doing it manually.
+    await gu.setType('Reference List', {apply: true});
+    await gu.setType('Reference', {apply: true});
+    await gu.checkForErrors();
+
+    await revert();
+  });
+});
+
+async function addReverseColumn(tableId: string, colId: string) {
+  await gu.sendActions([
+    ['AddReverseColumn', tableId, colId],
+  ]);
+}
+
+/**
+ * Returns an array of column headers for each table in the document.
+ */
+async function columns() {
+  const headers: string[][] = [];
+
+  for (const table of await driver.findAll('.gridview_stick-top')) {
+    const cols = await table.findAll('.g-column-label', e => e.getText());
+    headers.push(cols);
+  }
+  return headers;
+}
--- a/test/nbrowser/gristUtils.ts
+++ b/test/nbrowser/gristUtils.ts
@@ -18,7 +18,7 @@ import { AccessLevel } from 'app/common/CustomWidget';
 import { decodeUrl } from 'app/common/gristUrls';
 import { FullUser, UserProfile } from 'app/common/LoginSessionAPI';
 import { resetOrg } from 'app/common/resetOrg';
-import { UserAction } from 'app/common/DocActions';
+import { DocAction, UserAction } from 'app/common/DocActions';
 import { TestState } from 'app/common/TestState';
 import { Organization as APIOrganization, DocStateComparison,
         UserAPI, UserAPIImpl, Workspace } from 'app/common/UserAPI';
@@ -342,6 +342,31 @@ export async function getVisibleGridCells<T>(
  return rowNums.map((n) => fields[visibleRowNums.indexOf(n)]);
 }

+/**
+ * Checks if visible section with a grid contains the given data.
+ * Data is in form of:
+ * [0, "ColA", "ColB"]
+ * [1, "Val",  "1"] // cells are strings
+ * [2, "Val2", "2"]
+ */
+export async function assertGridData(section: string, data: any[][]) {
+  // Data is in form of
+  // [0, "ColA", "ColB"]
+  // [1, "Val",  1]
+
+  const rowIndices = data.slice(1).map((row: number[]) => row[0]);
+  const columnNames = data[0].slice(1);
+
+  for(const col of columnNames) {
+    const colIndex = columnNames.indexOf(col) + 1;
+    const colValues = data.slice(1).map((row: string[]) => row[colIndex]);
+    assert.deepEqual(
+      await getVisibleGridCells(col, rowIndices, section),
+      colValues
+    );
+  }
+}
+
 /**
 * Experimental fast version of getVisibleGridCells that reads data directly from browser by
 * invoking javascript code.
@@ -539,7 +564,8 @@ export function getColumnHeader(colOrColOptions: string|IColHeader): WebElementP
 }

 export async function getColumnNames() {
-  return (await driver.findAll('.column_name', el => el.getText()))
+  const section = await driver.findWait('.active_section', 4000);
+  return (await section.findAll('.column_name', el => el.getText()))
    .filter(name => name !== '+');
 }

@@ -1028,7 +1054,7 @@ export async function waitForLabelInput(): Promise<void> {
 /**
 * Sends UserActions using client api from the browser.
 */
-export async function sendActions(actions: UserAction[]) {
+export async function sendActions(actions: (DocAction|UserAction)[]) {
  await driver.manage().setTimeouts({
    script: 1000 * 2, /* 2 seconds, default is 0.5s */
  });
@@ -1438,6 +1464,13 @@ export async function redo(optCount: number = 1, optTimeout?: number) {
  await waitForServer(optTimeout);
 }

+export async function redoAll() {
+  const isActive = () => driver.find('.test-redo').matches('[class*="disabled"]').then((v) => !v);
+  while (await isActive()) {
+    await redo();
+  }
+}
+
 /**
 * Asserts the absence of javascript errors.
 */
@@ -1458,7 +1491,10 @@ export async function openWidgetPanel(tab: 'widget'|'sortAndFilter'|'data' = 'wi
 /**
 * Opens a Creator Panel on Widget/Table settings tab.
 */
- export async function openColumnPanel() {
+ export async function openColumnPanel(col?: string|number) {
+  if (col !== undefined) {
+    await getColumnHeader({col}).click();
+  }
  await toggleSidePanel('right', 'open');
  await driver.find('.test-right-tab-field').click();
 }
@@ -1746,7 +1782,17 @@ export async function selectAllKey() {
 * Send keys, with support for Key.chord(), similar to driver.sendKeys(). Note that while
 * elem.sendKeys() supports Key.chord(...), driver.sendKeys() does not. This is a replacement.
 */
-export async function sendKeys(...keys: string[]) {
+export async function sendKeys(...keys: string[]): Promise<void>
+/**
+ * Send keys with a pause between each key.
+ */
+export async function sendKeys(interval: number, ...keys: string[]): Promise<void>
+export async function sendKeys(...args: (string|number)[]) {
+  let interval = 0;
+  if (typeof args[0] === 'number') {
+    interval = args.shift() as number;
+  }
+  const keys = args as string[];
  // tslint:disable-next-line:max-line-length
  // Implementation follows the description of WebElement.sendKeys functionality at https://github.com/SeleniumHQ/selenium/blob/2f7727c314f943582f9f1b2a7e4d77ebdd64bdd3/javascript/node/selenium-webdriver/lib/webdriver.js#L2146
  await driver.withActions((a) => {
@@ -1761,19 +1807,19 @@ export async function sendKeys(...keys: string[]) {
        } else {
          a.sendKeys(key);
        }
+        if (interval) {
+          a.pause(interval);
+        }
      }
    }
  });
 }

 /**
- * Send keys with a pause between each key.
+ * An default ovveride for sendKeys that sends keys slowly, suitable for formula editor.
 */
-export async function sendKeysSlowly(keys: string[], delayMs = 40) {
-  for (const [i, key] of keys.entries()) {
-    await sendKeys(key);
-    if (i < keys.length - 1) { await driver.sleep(delayMs); }
-  }
+export async function sendKeysSlowly(...keys: string[]) {
+  return await sendKeys(10, ...keys);
 }

 /**