diff --git a/sandbox/grist/functions/info.py b/sandbox/grist/functions/info.py index 0d01466c..b41722b1 100644 --- a/sandbox/grist/functions/info.py +++ b/sandbox/grist/functions/info.py @@ -7,9 +7,10 @@ import math import numbers import re +import column from functions import date # pylint: disable=import-error from usertypes import AltText # pylint: disable=import-error -from records import Record # pylint: disable=import-error +from records import Record, RecordSet def ISBLANK(value): """ @@ -499,6 +500,86 @@ def CELL(info_type, reference): raise NotImplementedError() +def RECORD(record_or_list, dates_as_iso=True, expand_refs=0): + """ + Returns a Python dictionary with all fields in the given record. If a list of records is given, + returns a list of corresponding Python dictionaries. + + If dates_as_iso is set (which is the default), Date and DateTime values are converted to string + using ISO 8601 format. + + If expand_refs is set to 1 or higher, Reference values are replaced with a RECORD representation + of the referenced record, expanding the given number of levels. + + Error values present in cells of the record are replaced with None value, and a special key of + "_error_" gets added containing the error messages for those cells. For example: + `{"Ratio": None, "_error_": {"Ratio": "ZeroDivisionError: integer division or modulo by zero"}}` + + Note that care is needed to avoid circular references when using RECORD(), since it creates a + dependency on every cell in the record. In case of RECORD(rec), the cell containing this call + will be omitted from the resulting dictionary. + + For example: + ``` + RECORD($Person) + RECORD(rec) + RECORD(People.lookupOne(First_Name="Alice")) + RECORD(People.lookupRecords(Department="HR")) + ``` + """ + if isinstance(record_or_list, Record): + return _prepare_record_dict(record_or_list, dates_as_iso=dates_as_iso, expand_refs=expand_refs) + + try: + records = list(record_or_list) + assert all(isinstance(r, Record) for r in records) + except Exception: + raise ValueError('RECORD() requires a Record or an iterable of Records') + + return [_prepare_record_dict(r, dates_as_iso=dates_as_iso, expand_refs=expand_refs) + for r in records] + + +def _prepare_record_dict(record, dates_as_iso=True, expand_refs=0): + table_id = record._table.table_id + docmodel = record._table._engine.docmodel + columns = docmodel.get_table_rec(table_id).columns + frame = record._table._engine.get_current_frame() + + result = {'id': int(record)} + errors = {} + for col in columns: + col_id = col.colId + # Skip helper columns. + if not column.is_visible_column(col_id): + continue + + # Avoid trying to access the cell being evaluated, since cycles get detected even if the + # CircularRef exception is caught. TODO This is hacky, and imperfect. If another column + # references a column containing the RECORD(rec) call, CircularRefError will still happen. + if frame and frame.node == (table_id, col_id): + continue + + try: + val = getattr(record, col_id) + if dates_as_iso and isinstance(val, datetime.date): + val = val.isoformat() + elif expand_refs and isinstance(val, (Record, RecordSet)): + # Reduce expand_refs levels. + if val: + val = RECORD(val, dates_as_iso=dates_as_iso, expand_refs=expand_refs - 1) + else: + val = None + result[col_id] = val + except Exception as e: + result[col_id] = None + errors[col_id] = "%s: %s" % (type(e).__name__, str(e)) + + if errors: + result["_error_"] = errors + return result + + # Unique sentinel value to represent that a lazy value evaluates with an exception. _error_sentinel = object() diff --git a/sandbox/grist/test_record_func.py b/sandbox/grist/test_record_func.py new file mode 100644 index 00000000..2f540464 --- /dev/null +++ b/sandbox/grist/test_record_func.py @@ -0,0 +1,196 @@ +import datetime +import actions +import logger +import moment +import objtypes + +import testsamples +import testutil +import test_engine + +log = logger.Logger(__name__, logger.INFO) + +def _bulk_update(table_name, col_names, row_data): + return actions.BulkUpdateRecord( + *testutil.table_data_from_rows(table_name, col_names, row_data)) + +class TestRecordFunc(test_engine.EngineTestCase): + + def test_record_self(self): + self.load_sample(testsamples.sample_students) + self.add_column("Schools", "Foo", formula='repr(RECORD(rec))') + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"], + [2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"], + [3, "{'address': Address[13], 'id': 3, 'name': 'Yale'}"], + [4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"], + ]) + + # A change to data is reflected + self.update_record("Schools", 3, name="UConn") + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"], + [2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"], + [3, "{'address': Address[13], 'id': 3, 'name': 'UConn'}"], + [4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"], + ]) + + # A column addition is reflected + self.add_column("Schools", "Bar", formula='len($name)') + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'address': Address[11], 'Bar': 8, 'id': 1, 'name': 'Columbia'}"], + [2, "{'address': Address[12], 'Bar': 8, 'id': 2, 'name': 'Columbia'}"], + [3, "{'address': Address[13], 'Bar': 5, 'id': 3, 'name': 'UConn'}"], + [4, "{'address': Address[14], 'Bar': 4, 'id': 4, 'name': 'Yale'}"], + ]) + + def test_reference(self): + self.load_sample(testsamples.sample_students) + self.add_column("Schools", "Foo", formula='repr(RECORD($address))') + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'city': 'New York', 'id': 11}"], + [2, "{'city': 'Colombia', 'id': 12}"], + [3, "{'city': 'New Haven', 'id': 13}"], + [4, "{'city': 'West Haven', 'id': 14}"], + ]) + + # A change to referenced data is still reflected; try a different kind of change here + self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"]) + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'ciudad': 'New York', 'id': 11}"], + [2, "{'ciudad': 'Colombia', 'id': 12}"], + [3, "{'ciudad': 'New Haven', 'id': 13}"], + [4, "{'ciudad': 'West Haven', 'id': 14}"], + ]) + + def test_record_expand_refs(self): + self.load_sample(testsamples.sample_students) + self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))') + self.add_column("Address", "student", type="Ref:Students") + self.update_record("Address", 12, student=6) + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'address': {'city': 'New York', 'id': 11, 'student': Students[0]}," + + " 'id': 1, 'name': 'Columbia'}"], + [2, "{'address': {'city': 'Colombia', 'id': 12, 'student': Students[6]}," + + " 'id': 2, 'name': 'Columbia'}"], + [3, "{'address': {'city': 'New Haven', 'id': 13, 'student': Students[0]}," + + " 'id': 3, 'name': 'Yale'}"], + [4, "{'address': {'city': 'West Haven', 'id': 14, 'student': Students[0]}," + + " 'id': 4, 'name': 'Yale'}"], + ]) + + self.modify_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=2))') + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'address': {'city': 'New York', 'id': 11, 'student': None}," + + " 'id': 1, 'name': 'Columbia'}"], + [2, "{'address': {'city': 'Colombia', 'id': 12, " + + "'student': {'firstName': 'Gerald', 'schoolName': 'Yale', 'lastName': 'Ford', " + + "'schoolCities': 'New Haven:West Haven', 'schoolIds': '3:4', 'id': 6}}," + + " 'id': 2, 'name': 'Columbia'}"], + [3, "{'address': {'city': 'New Haven', 'id': 13, 'student': None}," + + " 'id': 3, 'name': 'Yale'}"], + [4, "{'address': {'city': 'West Haven', 'id': 14, 'student': None}," + + " 'id': 4, 'name': 'Yale'}"], + ]) + + def test_record_date_options(self): + self.load_sample(testsamples.sample_students) + self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))') + self.add_column("Address", "DT", type='DateTime') + self.add_column("Address", "D", type='Date', formula="$DT and $DT.date()") + self.update_records("Address", ['id', 'DT'], [ + [11, 1600000000], + [13, 1500000000], + ]) + + d1 = datetime.datetime(2020, 9, 13, 8, 26, 40, tzinfo=moment.tzinfo('America/New_York')) + d2 = datetime.datetime(2017, 7, 13, 22, 40, tzinfo=moment.tzinfo('America/New_York')) + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " % + (d1.isoformat(), d1.date().isoformat()) + + "'id': 1, 'name': 'Columbia'}"], + [2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " + + "'id': 2, 'name': 'Columbia'}"], + [3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " % + (d2.isoformat(), d2.date().isoformat()) + + "'id': 3, 'name': 'Yale'}"], + [4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " + + "'id': 4, 'name': 'Yale'}"], + ]) + + self.modify_column("Schools", "Foo", + formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=False))') + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'address': {'city': 'New York', 'DT': %s, 'id': 11, 'D': %s}, " % + (repr(d1), repr(d1.date())) + + "'id': 1, 'name': 'Columbia'}"], + [2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " + + "'id': 2, 'name': 'Columbia'}"], + [3, "{'address': {'city': 'New Haven', 'DT': %s, 'id': 13, 'D': %s}, " % + (repr(d2), repr(d2.date())) + + "'id': 3, 'name': 'Yale'}"], + [4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " + + "'id': 4, 'name': 'Yale'}"], + ]) + + def test_record_set(self): + self.load_sample(testsamples.sample_students) + self.add_column("Students", "schools", formula='Schools.lookupRecords(name=$schoolName)') + self.add_column("Students", "Foo", formula='repr(RECORD($schools))') + self.assertPartialData("Students", ["id", "Foo"], [ + [1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," + + " {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"], + [2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," + + " {'address': Address[14], 'id': 4, 'name': 'Yale'}]"], + [3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," + + " {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"], + [4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," + + " {'address': Address[14], 'id': 4, 'name': 'Yale'}]"], + [5, "[]"], + [6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," + + " {'address': Address[14], 'id': 4, 'name': 'Yale'}]"], + ]) + + # Try a field with filtered lookupRecords result, as an iterable. + self.modify_column("Students", "Foo", + formula='repr(RECORD(s for s in $schools if s.address.city.startswith("New")))') + self.assertPartialData("Students", ["id", "Foo"], [ + [1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"], + [2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"], + [3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"], + [4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"], + [5, "[]"], + [6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"], + ]) + + def test_record_bad_calls(self): + self.load_sample(testsamples.sample_students) + self.add_column("Schools", "Foo", formula='repr(RECORD($name))') + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, objtypes.RaisedException(ValueError())], + [2, objtypes.RaisedException(ValueError())], + [3, objtypes.RaisedException(ValueError())], + [4, objtypes.RaisedException(ValueError())], + ]) + self.modify_column("Schools", "Foo", formula='repr(RECORD([rec] if $id == 2 else $id))') + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, objtypes.RaisedException(ValueError())], + [2, "[{'address': Address[12], 'id': 2, 'name': 'Columbia'}]"], + [3, objtypes.RaisedException(ValueError())], + [4, objtypes.RaisedException(ValueError())], + ]) + self.assertEqual(self.engine.get_formula_error('Schools', 'Foo', 1).error.message, + 'RECORD() requires a Record or an iterable of Records') + + def test_record_error_cells(self): + self.load_sample(testsamples.sample_students) + self.add_column("Schools", "Foo", formula='repr(RECORD($address))') + self.add_column("Address", "Bar", formula='$id/($id%2)') + self.assertPartialData("Schools", ["id", "Foo"], [ + [1, "{'city': 'New York', 'Bar': 11, 'id': 11}"], + [2, "{'city': 'Colombia', 'Bar': None, 'id': 12, " + + "'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"], + [3, "{'city': 'New Haven', 'Bar': 13, 'id': 13}"], + [4, "{'city': 'West Haven', 'Bar': None, 'id': 14, " + + "'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"], + ])