(core) Implement RECORD function that converts Records to Python dictionaries

Summary:
Examining a Record is currently difficult, because its columns are hard to
list (and to use), and CircularRef errors hard to avoid. The RECORD function
takes care of this mess to return a simple dictionary of values.

- Supports dates_as_iso=False flag to turn off the translation of date/datetime
  objects to strings.
- Supports expand_refs=True flag to apply RECORD() to encountered values of
  type Record, for a single level of nesting.

Test Plan: Added a unittest for RECORD()

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2582

sandbox/grist/functions/info.py

@@ -7,9 +7,10 @@ import math
 import numbers
 import re
 
+import column
 from functions import date      # pylint: disable=import-error
 from usertypes import AltText   # pylint: disable=import-error
-from records import Record      # pylint: disable=import-error
+from records import Record, RecordSet
 
 def ISBLANK(value):
   """
@@ -499,6 +500,86 @@ def CELL(info_type, reference):
   raise NotImplementedError()
 
 
+def RECORD(record_or_list, dates_as_iso=True, expand_refs=0):
+  """
+  Returns a Python dictionary with all fields in the given record. If a list of records is given,
+  returns a list of corresponding Python dictionaries.
+
+  If dates_as_iso is set (which is the default), Date and DateTime values are converted to string
+  using ISO 8601 format.
+
+  If expand_refs is set to 1 or higher, Reference values are replaced with a RECORD representation
+  of the referenced record, expanding the given number of levels.
+
+  Error values present in cells of the record are replaced with None value, and a special key of
+  "_error_" gets added containing the error messages for those cells. For example:
+  `{"Ratio": None, "_error_": {"Ratio": "ZeroDivisionError: integer division or modulo by zero"}}`
+
+  Note that care is needed to avoid circular references when using RECORD(), since it creates a
+  dependency on every cell in the record. In case of RECORD(rec), the cell containing this call
+  will be omitted from the resulting dictionary.
+
+  For example:
+  ```
+  RECORD($Person)
+  RECORD(rec)
+  RECORD(People.lookupOne(First_Name="Alice"))
+  RECORD(People.lookupRecords(Department="HR"))
+  ```
+  """
+  if isinstance(record_or_list, Record):
+    return _prepare_record_dict(record_or_list, dates_as_iso=dates_as_iso, expand_refs=expand_refs)
+
+  try:
+    records = list(record_or_list)
+    assert all(isinstance(r, Record) for r in records)
+  except Exception:
+    raise ValueError('RECORD() requires a Record or an iterable of Records')
+
+  return [_prepare_record_dict(r, dates_as_iso=dates_as_iso, expand_refs=expand_refs)
+          for r in records]
+
+
+def _prepare_record_dict(record, dates_as_iso=True, expand_refs=0):
+  table_id = record._table.table_id
+  docmodel = record._table._engine.docmodel
+  columns = docmodel.get_table_rec(table_id).columns
+  frame = record._table._engine.get_current_frame()
+
+  result = {'id': int(record)}
+  errors = {}
+  for col in columns:
+    col_id = col.colId
+    # Skip helper columns.
+    if not column.is_visible_column(col_id):
+      continue
+
+    # Avoid trying to access the cell being evaluated, since cycles get detected even if the
+    # CircularRef exception is caught. TODO This is hacky, and imperfect. If another column
+    # references a column containing the RECORD(rec) call, CircularRefError will still happen.
+    if frame and frame.node == (table_id, col_id):
+      continue
+
+    try:
+      val = getattr(record, col_id)
+      if dates_as_iso and isinstance(val, datetime.date):
+        val = val.isoformat()
+      elif expand_refs and isinstance(val, (Record, RecordSet)):
+        # Reduce expand_refs levels.
+        if val:
+          val = RECORD(val, dates_as_iso=dates_as_iso, expand_refs=expand_refs - 1)
+        else:
+          val = None
+      result[col_id] = val
+    except Exception as e:
+      result[col_id] = None
+      errors[col_id] = "%s: %s" % (type(e).__name__, str(e))
+
+  if errors:
+    result["_error_"] = errors
+  return result
+
+
 # Unique sentinel value to represent that a lazy value evaluates with an exception.
 _error_sentinel = object()
 

sandbox/grist/test_record_func.py

@@ -0,0 +1,196 @@
+import datetime
+import actions
+import logger
+import moment
+import objtypes
+
+import testsamples
+import testutil
+import test_engine
+
+log = logger.Logger(__name__, logger.INFO)
+
+def _bulk_update(table_name, col_names, row_data):
+  return actions.BulkUpdateRecord(
+    *testutil.table_data_from_rows(table_name, col_names, row_data))
+
+class TestRecordFunc(test_engine.EngineTestCase):
+
+  def test_record_self(self):
+    self.load_sample(testsamples.sample_students)
+    self.add_column("Schools", "Foo", formula='repr(RECORD(rec))')
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1,     "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"],
+      [2,     "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"],
+      [3,     "{'address': Address[13], 'id': 3, 'name': 'Yale'}"],
+      [4,     "{'address': Address[14], 'id': 4, 'name': 'Yale'}"],
+    ])
+
+    # A change to data is reflected
+    self.update_record("Schools", 3, name="UConn")
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1,     "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"],
+      [2,     "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"],
+      [3,     "{'address': Address[13], 'id': 3, 'name': 'UConn'}"],
+      [4,     "{'address': Address[14], 'id': 4, 'name': 'Yale'}"],
+    ])
+
+    # A column addition is reflected
+    self.add_column("Schools", "Bar", formula='len($name)')
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1,     "{'address': Address[11], 'Bar': 8, 'id': 1, 'name': 'Columbia'}"],
+      [2,     "{'address': Address[12], 'Bar': 8, 'id': 2, 'name': 'Columbia'}"],
+      [3,     "{'address': Address[13], 'Bar': 5, 'id': 3, 'name': 'UConn'}"],
+      [4,     "{'address': Address[14], 'Bar': 4, 'id': 4, 'name': 'Yale'}"],
+    ])
+
+  def test_reference(self):
+    self.load_sample(testsamples.sample_students)
+    self.add_column("Schools", "Foo", formula='repr(RECORD($address))')
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1,     "{'city': 'New York', 'id': 11}"],
+      [2,     "{'city': 'Colombia', 'id': 12}"],
+      [3,     "{'city': 'New Haven', 'id': 13}"],
+      [4,     "{'city': 'West Haven', 'id': 14}"],
+    ])
+
+    # A change to referenced data is still reflected; try a different kind of change here
+    self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1,     "{'ciudad': 'New York', 'id': 11}"],
+      [2,     "{'ciudad': 'Colombia', 'id': 12}"],
+      [3,     "{'ciudad': 'New Haven', 'id': 13}"],
+      [4,     "{'ciudad': 'West Haven', 'id': 14}"],
+    ])
+
+  def test_record_expand_refs(self):
+    self.load_sample(testsamples.sample_students)
+    self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))')
+    self.add_column("Address", "student", type="Ref:Students")
+    self.update_record("Address", 12, student=6)
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1, "{'address': {'city': 'New York', 'id': 11, 'student': Students[0]}," +
+        " 'id': 1, 'name': 'Columbia'}"],
+      [2, "{'address': {'city': 'Colombia', 'id': 12, 'student': Students[6]}," +
+        " 'id': 2, 'name': 'Columbia'}"],
+      [3, "{'address': {'city': 'New Haven', 'id': 13, 'student': Students[0]}," +
+        " 'id': 3, 'name': 'Yale'}"],
+      [4, "{'address': {'city': 'West Haven', 'id': 14, 'student': Students[0]}," +
+        " 'id': 4, 'name': 'Yale'}"],
+    ])
+
+    self.modify_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=2))')
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1, "{'address': {'city': 'New York', 'id': 11, 'student': None}," +
+        " 'id': 1, 'name': 'Columbia'}"],
+      [2, "{'address': {'city': 'Colombia', 'id': 12, " +
+        "'student': {'firstName': 'Gerald', 'schoolName': 'Yale', 'lastName': 'Ford', " +
+        "'schoolCities': 'New Haven:West Haven', 'schoolIds': '3:4', 'id': 6}}," +
+        " 'id': 2, 'name': 'Columbia'}"],
+      [3, "{'address': {'city': 'New Haven', 'id': 13, 'student': None}," +
+        " 'id': 3, 'name': 'Yale'}"],
+      [4, "{'address': {'city': 'West Haven', 'id': 14, 'student': None}," +
+        " 'id': 4, 'name': 'Yale'}"],
+    ])
+
+  def test_record_date_options(self):
+    self.load_sample(testsamples.sample_students)
+    self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))')
+    self.add_column("Address", "DT", type='DateTime')
+    self.add_column("Address", "D", type='Date', formula="$DT and $DT.date()")
+    self.update_records("Address", ['id', 'DT'], [
+      [11, 1600000000],
+      [13, 1500000000],
+    ])
+
+    d1 = datetime.datetime(2020, 9, 13, 8, 26, 40, tzinfo=moment.tzinfo('America/New_York'))
+    d2 = datetime.datetime(2017, 7, 13, 22, 40, tzinfo=moment.tzinfo('America/New_York'))
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " %
+          (d1.isoformat(), d1.date().isoformat()) +
+          "'id': 1, 'name': 'Columbia'}"],
+      [2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
+          "'id': 2, 'name': 'Columbia'}"],
+      [3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " %
+          (d2.isoformat(), d2.date().isoformat()) +
+          "'id': 3, 'name': 'Yale'}"],
+      [4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
+          "'id': 4, 'name': 'Yale'}"],
+    ])
+
+    self.modify_column("Schools", "Foo",
+        formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=False))')
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1, "{'address': {'city': 'New York', 'DT': %s, 'id': 11, 'D': %s}, " %
+          (repr(d1), repr(d1.date())) +
+          "'id': 1, 'name': 'Columbia'}"],
+      [2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
+          "'id': 2, 'name': 'Columbia'}"],
+      [3, "{'address': {'city': 'New Haven', 'DT': %s, 'id': 13, 'D': %s}, " %
+          (repr(d2), repr(d2.date())) +
+          "'id': 3, 'name': 'Yale'}"],
+      [4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
+          "'id': 4, 'name': 'Yale'}"],
+    ])
+
+  def test_record_set(self):
+    self.load_sample(testsamples.sample_students)
+    self.add_column("Students", "schools", formula='Schools.lookupRecords(name=$schoolName)')
+    self.add_column("Students", "Foo", formula='repr(RECORD($schools))')
+    self.assertPartialData("Students", ["id", "Foo"], [
+      [1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," +
+          " {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
+      [2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
+          " {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
+      [3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," +
+          " {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
+      [4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
+          " {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
+      [5, "[]"],
+      [6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
+          " {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
+    ])
+
+    # Try a field with filtered lookupRecords result, as an iterable.
+    self.modify_column("Students", "Foo",
+        formula='repr(RECORD(s for s in $schools if s.address.city.startswith("New")))')
+    self.assertPartialData("Students", ["id", "Foo"], [
+      [1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"],
+      [2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
+      [3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"],
+      [4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
+      [5, "[]"],
+      [6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
+    ])
+
+  def test_record_bad_calls(self):
+    self.load_sample(testsamples.sample_students)
+    self.add_column("Schools", "Foo", formula='repr(RECORD($name))')
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1, objtypes.RaisedException(ValueError())],
+      [2, objtypes.RaisedException(ValueError())],
+      [3, objtypes.RaisedException(ValueError())],
+      [4, objtypes.RaisedException(ValueError())],
+    ])
+    self.modify_column("Schools", "Foo", formula='repr(RECORD([rec] if $id == 2 else $id))')
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1, objtypes.RaisedException(ValueError())],
+      [2, "[{'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
+      [3, objtypes.RaisedException(ValueError())],
+      [4, objtypes.RaisedException(ValueError())],
+    ])
+    self.assertEqual(self.engine.get_formula_error('Schools', 'Foo', 1).error.message,
+        'RECORD() requires a Record or an iterable of Records')
+
+  def test_record_error_cells(self):
+    self.load_sample(testsamples.sample_students)
+    self.add_column("Schools", "Foo", formula='repr(RECORD($address))')
+    self.add_column("Address", "Bar", formula='$id/($id%2)')
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1,     "{'city': 'New York', 'Bar': 11, 'id': 11}"],
+      [2,     "{'city': 'Colombia', 'Bar': None, 'id': 12, " +
+              "'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"],
+      [3,     "{'city': 'New Haven', 'Bar': 13, 'id': 13}"],
+      [4,     "{'city': 'West Haven', 'Bar': None, 'id': 14, " +
+              "'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"],
+    ])