(core) Parsing multiple values in reflists, parsing refs without table data in client

Summary:
Added a new object type code `l` (for lookup) which can be used in user actions as a temporary cell value in ref[list] columns and is immediately converted to a row ID in the data engine. The value contains the original raw string (to be used as alt text), the column ID to lookup (typically the visible column) and one or more values to lookup.

For reflists, valueParser now tries parsing the string first as JSON, then as a CSV row, and applies the visible column parsed to each item.

Both ref and reflists columns no longer format the parsed value when there's no matching reference, the original unparsed string is used as alttext instead.

Test Plan: Added another table "Multi-References" to CopyPaste test. Made that table and the References table test with and without table data loaded in the browser.

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D3118

sandbox/grist/column.py

@@ -158,8 +158,15 @@ class BaseColumn(object):
         raise raw.error
       else:
         raise objtypes.CellError(self.table_id, self.col_id, row_id, raw.error)
+
+    return self._convert_raw_value(raw)
+
+  def _convert_raw_value(self, raw):
     if self.type_obj.is_right_type(raw):
       return self._make_rich_value(raw)
+    return self._alt_text(raw)
+
+  def _alt_text(self, raw):
     return usertypes.AltText(str(raw), self.type_obj.typename())
 
   def _make_rich_value(self, typed_value):
@@ -440,6 +447,16 @@ class BaseReferenceColumn(BaseColumn):
     """
     return self.getdefault()
 
+  def _lookup(self, reference_lookup, value):
+    col_id = (
+        reference_lookup.options.get("column")
+        or self._target_table._engine.docmodel
+            .get_column_rec(self.table_id, self.col_id).visibleCol.colId
+        or "id"
+    )
+    target_value = self._target_table.get_column(col_id)._convert_raw_value(value)
+    return self._target_table.lookup_one_record(**{col_id: target_value})
+
 
 class ReferenceColumn(BaseReferenceColumn):
   """
@@ -465,6 +482,11 @@ class ReferenceColumn(BaseReferenceColumn):
       values = action_summary.translate_new_row_ids(self._target_table.table_id, values)
     return values, []
 
+  def convert(self, val):
+    if isinstance(val, objtypes.ReferenceLookup):
+      val = self._lookup(val, val.value) or self._alt_text(val.alt_text)
+    return super(ReferenceColumn, self).convert(val)
+
 
 class ReferenceListColumn(BaseReferenceColumn):
   """
@@ -502,6 +524,21 @@ class ReferenceListColumn(BaseReferenceColumn):
       raw = [r for r in raw if r not in target_row_ids] or None
     return raw
 
+  def convert(self, val):
+    if isinstance(val, objtypes.ReferenceLookup):
+      result = []
+      values = val.value
+      if not isinstance(values, list):
+        values = [values]
+      for value in values:
+        lookup_value = self._lookup(val, value)
+        if not lookup_value:
+          return self._alt_text(val.alt_text)
+        result.append(lookup_value)
+      val = result
+    return super(ReferenceListColumn, self).convert(val)
+
+
 # Set up the relationship between usertypes objects and column objects.
 usertypes.BaseColumnType.ColType = DataColumn
 usertypes.Reference.ColType = ReferenceColumn

sandbox/grist/objtypes.py

@@ -232,6 +232,8 @@ def decode_object(value):
       return RaisedException.decode_args(*args)
     elif code == 'L':
       return [decode_object(item) for item in args]
+    elif code == 'l':
+      return ReferenceLookup(*args)
     elif code == 'O':
       return {decode_object(key): decode_object(val) for key, val in six.iteritems(args[0])}
     elif code == 'P':
@@ -381,3 +383,19 @@ class RecordSetStub(object):
   def __init__(self, table_id, row_ids):
     self.table_id = table_id
     self.row_ids = row_ids
+
+
+class ReferenceLookup(object):
+  def __init__(self, value, options=None):
+    self.value = value
+    self.options = options or {}
+
+  @property
+  def alt_text(self):
+    result = self.options.get("raw")
+    if result is None:
+      values = self.value
+      if not isinstance(values, list):
+        values = [values]
+      result = ", ".join(map(six.text_type, values))
+    return result

sandbox/grist/test_useractions.py

@@ -867,3 +867,95 @@ class TestUserActions(test_engine.EngineTestCase):
       [11, 5],
       [12, [[]]],
     ])
+
+  def test_reference_lookup(self):
+    sample = testutil.parse_test_sample({
+      "SCHEMA": [
+        [1, "Table1", [
+          [1, "name",    "Text",           False, "", "name",    ""],
+          [2, "ref",     "Ref:Table1",     False, "", "ref",     ""],
+          [3, "reflist", "RefList:Table1", False, "", "reflist", ""],
+        ]],
+      ],
+      "DATA": {
+        "Table1": [
+          ["id", "name"],
+          [1, "a"],
+          [2, "b"],
+        ],
+      }
+    })
+    self.load_sample(sample)
+    self.update_record("_grist_Tables_column", 2, visibleCol=1)
+
+    # Normal case
+    out_actions = self.apply_user_action(
+      ["UpdateRecord", "Table1", 1, {"ref": ["l", "b", {"column": "name"}]}])
+    self.assertPartialOutActions(out_actions, {'stored': [
+      ["UpdateRecord", "Table1", 1, {"ref": 2}]]})
+
+    # Use ref.visibleCol (name) as default lookup column
+    out_actions = self.apply_user_action(
+      ["UpdateRecord", "Table1", 2, {"ref": ["l", "a"]}])
+    self.assertPartialOutActions(out_actions, {'stored': [
+      ["UpdateRecord", "Table1", 2, {"ref": 1}]]})
+
+    # No match found, generate alttext from value
+    out_actions = self.apply_user_action(
+      ["UpdateRecord", "Table1", 2, {"ref": ["l", "foo", {"column": "name"}]}])
+    self.assertPartialOutActions(out_actions, {'stored': [
+      ["UpdateRecord", "Table1", 2, {"ref": "foo"}]]})
+
+    # No match found, use provided alttext
+    out_actions = self.apply_user_action(
+      ["UpdateRecord", "Table1", 2, {"ref": ["l", "foo", {"column": "name", "raw": "alt"}]}])
+    self.assertPartialOutActions(out_actions, {'stored': [
+      ["UpdateRecord", "Table1", 2, {"ref": "alt"}]]})
+
+    # Normal case, adding instead of updating
+    out_actions = self.apply_user_action(
+      ["AddRecord", "Table1", 3,
+       {"ref": ["l", "b", {"column": "name"}],
+        "name": "c"}])
+    self.assertPartialOutActions(out_actions, {'stored': [
+      ["AddRecord", "Table1", 3,
+       {"ref": 2,
+        "name": "c"}]]})
+
+    # Testing reflist and bulk action
+    out_actions = self.apply_user_action(
+      ["BulkUpdateRecord", "Table1", [1, 2, 3],
+       {"reflist": [
+         ["l", "c", {"column": "name"}],  # value gets wrapped in list automatically
+         ["l", ["a", "b"], {"column": "name"}],  # normal case
+         # "a" matches but "foo" doesn't so the whole thing fails
+         ["l", ["a", "foo"], {"column": "name", "raw": "alt"}],
+       ]}])
+    self.assertPartialOutActions(out_actions, {'stored': [
+      ["BulkUpdateRecord", "Table1", [1, 2, 3],
+       {"reflist": [
+         ["L", 3],
+         ["L", 1, 2],
+         "alt",
+       ]}]]})
+
+    self.assertTableData('Table1', data=[
+      ["id", "name", "ref", "reflist"],
+      [1,    "a",    2,      [3]],
+      [2,    "b",    "alt",  [1, 2]],
+      [3,    "c",    2,      "alt"],
+    ])
+
+    # 'id' is used as the default visibleCol
+    out_actions = self.apply_user_action(
+      ["BulkUpdateRecord", "Table1", [1, 2],
+       {"reflist": [
+         ["l", 2],
+         ["l", 999],  # this row ID doesn't exist
+       ]}])
+    self.assertPartialOutActions(out_actions, {'stored': [
+      ["BulkUpdateRecord", "Table1", [1, 2],
+       {"reflist": [
+         ["L", 2],
+         "999",
+       ]}]]})