mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) Implement RECORD function that converts Records to Python dictionaries
Summary: Examining a Record is currently difficult, because its columns are hard to list (and to use), and CircularRef errors hard to avoid. The RECORD function takes care of this mess to return a simple dictionary of values. - Supports dates_as_iso=False flag to turn off the translation of date/datetime objects to strings. - Supports expand_refs=True flag to apply RECORD() to encountered values of type Record, for a single level of nesting. Test Plan: Added a unittest for RECORD() Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D2582
This commit is contained in:
Dmitry S
parent
4e20f7a8a2
commit
4e11b6a922
@ -7,9 +7,10 @@ import math
|
|||||||
import numbers
|
import numbers
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
import column
|
||||||
from functions import date # pylint: disable=import-error
|
from functions import date # pylint: disable=import-error
|
||||||
from usertypes import AltText # pylint: disable=import-error
|
from usertypes import AltText # pylint: disable=import-error
|
||||||
from records import Record # pylint: disable=import-error
|
from records import Record, RecordSet
|
||||||
|
|
||||||
def ISBLANK(value):
|
def ISBLANK(value):
|
||||||
"""
|
"""
|
||||||
@ -499,6 +500,86 @@ def CELL(info_type, reference):
|
|||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
|
||||||
|
def RECORD(record_or_list, dates_as_iso=True, expand_refs=0):
|
||||||
|
"""
|
||||||
|
Returns a Python dictionary with all fields in the given record. If a list of records is given,
|
||||||
|
returns a list of corresponding Python dictionaries.
|
||||||
|
|
||||||
|
If dates_as_iso is set (which is the default), Date and DateTime values are converted to string
|
||||||
|
using ISO 8601 format.
|
||||||
|
|
||||||
|
If expand_refs is set to 1 or higher, Reference values are replaced with a RECORD representation
|
||||||
|
of the referenced record, expanding the given number of levels.
|
||||||
|
|
||||||
|
Error values present in cells of the record are replaced with None value, and a special key of
|
||||||
|
"_error_" gets added containing the error messages for those cells. For example:
|
||||||
|
`{"Ratio": None, "_error_": {"Ratio": "ZeroDivisionError: integer division or modulo by zero"}}`
|
||||||
|
|
||||||
|
Note that care is needed to avoid circular references when using RECORD(), since it creates a
|
||||||
|
dependency on every cell in the record. In case of RECORD(rec), the cell containing this call
|
||||||
|
will be omitted from the resulting dictionary.
|
||||||
|
|
||||||
|
For example:
|
||||||
|
```
|
||||||
|
RECORD($Person)
|
||||||
|
RECORD(rec)
|
||||||
|
RECORD(People.lookupOne(First_Name="Alice"))
|
||||||
|
RECORD(People.lookupRecords(Department="HR"))
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
if isinstance(record_or_list, Record):
|
||||||
|
return _prepare_record_dict(record_or_list, dates_as_iso=dates_as_iso, expand_refs=expand_refs)
|
||||||
|
|
||||||
|
try:
|
||||||
|
records = list(record_or_list)
|
||||||
|
assert all(isinstance(r, Record) for r in records)
|
||||||
|
except Exception:
|
||||||
|
raise ValueError('RECORD() requires a Record or an iterable of Records')
|
||||||
|
|
||||||
|
return [_prepare_record_dict(r, dates_as_iso=dates_as_iso, expand_refs=expand_refs)
|
||||||
|
for r in records]
|
||||||
|
|
||||||
|
|
||||||
|
def _prepare_record_dict(record, dates_as_iso=True, expand_refs=0):
|
||||||
|
table_id = record._table.table_id
|
||||||
|
docmodel = record._table._engine.docmodel
|
||||||
|
columns = docmodel.get_table_rec(table_id).columns
|
||||||
|
frame = record._table._engine.get_current_frame()
|
||||||
|
|
||||||
|
result = {'id': int(record)}
|
||||||
|
errors = {}
|
||||||
|
for col in columns:
|
||||||
|
col_id = col.colId
|
||||||
|
# Skip helper columns.
|
||||||
|
if not column.is_visible_column(col_id):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Avoid trying to access the cell being evaluated, since cycles get detected even if the
|
||||||
|
# CircularRef exception is caught. TODO This is hacky, and imperfect. If another column
|
||||||
|
# references a column containing the RECORD(rec) call, CircularRefError will still happen.
|
||||||
|
if frame and frame.node == (table_id, col_id):
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
val = getattr(record, col_id)
|
||||||
|
if dates_as_iso and isinstance(val, datetime.date):
|
||||||
|
val = val.isoformat()
|
||||||
|
elif expand_refs and isinstance(val, (Record, RecordSet)):
|
||||||
|
# Reduce expand_refs levels.
|
||||||
|
if val:
|
||||||
|
val = RECORD(val, dates_as_iso=dates_as_iso, expand_refs=expand_refs - 1)
|
||||||
|
else:
|
||||||
|
val = None
|
||||||
|
result[col_id] = val
|
||||||
|
except Exception as e:
|
||||||
|
result[col_id] = None
|
||||||
|
errors[col_id] = "%s: %s" % (type(e).__name__, str(e))
|
||||||
|
|
||||||
|
if errors:
|
||||||
|
result["_error_"] = errors
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
# Unique sentinel value to represent that a lazy value evaluates with an exception.
|
# Unique sentinel value to represent that a lazy value evaluates with an exception.
|
||||||
_error_sentinel = object()
|
_error_sentinel = object()
|
||||||
|
|
||||||
|
|||||||
196
sandbox/grist/test_record_func.py
Normal file
196
sandbox/grist/test_record_func.py
Normal file
@ -0,0 +1,196 @@
|
|||||||
|
import datetime
|
||||||
|
import actions
|
||||||
|
import logger
|
||||||
|
import moment
|
||||||
|
import objtypes
|
||||||
|
|
||||||
|
import testsamples
|
||||||
|
import testutil
|
||||||
|
import test_engine
|
||||||
|
|
||||||
|
log = logger.Logger(__name__, logger.INFO)
|
||||||
|
|
||||||
|
def _bulk_update(table_name, col_names, row_data):
|
||||||
|
return actions.BulkUpdateRecord(
|
||||||
|
*testutil.table_data_from_rows(table_name, col_names, row_data))
|
||||||
|
|
||||||
|
class TestRecordFunc(test_engine.EngineTestCase):
|
||||||
|
|
||||||
|
def test_record_self(self):
|
||||||
|
self.load_sample(testsamples.sample_students)
|
||||||
|
self.add_column("Schools", "Foo", formula='repr(RECORD(rec))')
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"],
|
||||||
|
[2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"],
|
||||||
|
[3, "{'address': Address[13], 'id': 3, 'name': 'Yale'}"],
|
||||||
|
[4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"],
|
||||||
|
])
|
||||||
|
|
||||||
|
# A change to data is reflected
|
||||||
|
self.update_record("Schools", 3, name="UConn")
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"],
|
||||||
|
[2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"],
|
||||||
|
[3, "{'address': Address[13], 'id': 3, 'name': 'UConn'}"],
|
||||||
|
[4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"],
|
||||||
|
])
|
||||||
|
|
||||||
|
# A column addition is reflected
|
||||||
|
self.add_column("Schools", "Bar", formula='len($name)')
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'address': Address[11], 'Bar': 8, 'id': 1, 'name': 'Columbia'}"],
|
||||||
|
[2, "{'address': Address[12], 'Bar': 8, 'id': 2, 'name': 'Columbia'}"],
|
||||||
|
[3, "{'address': Address[13], 'Bar': 5, 'id': 3, 'name': 'UConn'}"],
|
||||||
|
[4, "{'address': Address[14], 'Bar': 4, 'id': 4, 'name': 'Yale'}"],
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_reference(self):
|
||||||
|
self.load_sample(testsamples.sample_students)
|
||||||
|
self.add_column("Schools", "Foo", formula='repr(RECORD($address))')
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'city': 'New York', 'id': 11}"],
|
||||||
|
[2, "{'city': 'Colombia', 'id': 12}"],
|
||||||
|
[3, "{'city': 'New Haven', 'id': 13}"],
|
||||||
|
[4, "{'city': 'West Haven', 'id': 14}"],
|
||||||
|
])
|
||||||
|
|
||||||
|
# A change to referenced data is still reflected; try a different kind of change here
|
||||||
|
self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'ciudad': 'New York', 'id': 11}"],
|
||||||
|
[2, "{'ciudad': 'Colombia', 'id': 12}"],
|
||||||
|
[3, "{'ciudad': 'New Haven', 'id': 13}"],
|
||||||
|
[4, "{'ciudad': 'West Haven', 'id': 14}"],
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_record_expand_refs(self):
|
||||||
|
self.load_sample(testsamples.sample_students)
|
||||||
|
self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))')
|
||||||
|
self.add_column("Address", "student", type="Ref:Students")
|
||||||
|
self.update_record("Address", 12, student=6)
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'address': {'city': 'New York', 'id': 11, 'student': Students[0]}," +
|
||||||
|
" 'id': 1, 'name': 'Columbia'}"],
|
||||||
|
[2, "{'address': {'city': 'Colombia', 'id': 12, 'student': Students[6]}," +
|
||||||
|
" 'id': 2, 'name': 'Columbia'}"],
|
||||||
|
[3, "{'address': {'city': 'New Haven', 'id': 13, 'student': Students[0]}," +
|
||||||
|
" 'id': 3, 'name': 'Yale'}"],
|
||||||
|
[4, "{'address': {'city': 'West Haven', 'id': 14, 'student': Students[0]}," +
|
||||||
|
" 'id': 4, 'name': 'Yale'}"],
|
||||||
|
])
|
||||||
|
|
||||||
|
self.modify_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=2))')
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'address': {'city': 'New York', 'id': 11, 'student': None}," +
|
||||||
|
" 'id': 1, 'name': 'Columbia'}"],
|
||||||
|
[2, "{'address': {'city': 'Colombia', 'id': 12, " +
|
||||||
|
"'student': {'firstName': 'Gerald', 'schoolName': 'Yale', 'lastName': 'Ford', " +
|
||||||
|
"'schoolCities': 'New Haven:West Haven', 'schoolIds': '3:4', 'id': 6}}," +
|
||||||
|
" 'id': 2, 'name': 'Columbia'}"],
|
||||||
|
[3, "{'address': {'city': 'New Haven', 'id': 13, 'student': None}," +
|
||||||
|
" 'id': 3, 'name': 'Yale'}"],
|
||||||
|
[4, "{'address': {'city': 'West Haven', 'id': 14, 'student': None}," +
|
||||||
|
" 'id': 4, 'name': 'Yale'}"],
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_record_date_options(self):
|
||||||
|
self.load_sample(testsamples.sample_students)
|
||||||
|
self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))')
|
||||||
|
self.add_column("Address", "DT", type='DateTime')
|
||||||
|
self.add_column("Address", "D", type='Date', formula="$DT and $DT.date()")
|
||||||
|
self.update_records("Address", ['id', 'DT'], [
|
||||||
|
[11, 1600000000],
|
||||||
|
[13, 1500000000],
|
||||||
|
])
|
||||||
|
|
||||||
|
d1 = datetime.datetime(2020, 9, 13, 8, 26, 40, tzinfo=moment.tzinfo('America/New_York'))
|
||||||
|
d2 = datetime.datetime(2017, 7, 13, 22, 40, tzinfo=moment.tzinfo('America/New_York'))
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " %
|
||||||
|
(d1.isoformat(), d1.date().isoformat()) +
|
||||||
|
"'id': 1, 'name': 'Columbia'}"],
|
||||||
|
[2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
|
||||||
|
"'id': 2, 'name': 'Columbia'}"],
|
||||||
|
[3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " %
|
||||||
|
(d2.isoformat(), d2.date().isoformat()) +
|
||||||
|
"'id': 3, 'name': 'Yale'}"],
|
||||||
|
[4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
|
||||||
|
"'id': 4, 'name': 'Yale'}"],
|
||||||
|
])
|
||||||
|
|
||||||
|
self.modify_column("Schools", "Foo",
|
||||||
|
formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=False))')
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'address': {'city': 'New York', 'DT': %s, 'id': 11, 'D': %s}, " %
|
||||||
|
(repr(d1), repr(d1.date())) +
|
||||||
|
"'id': 1, 'name': 'Columbia'}"],
|
||||||
|
[2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
|
||||||
|
"'id': 2, 'name': 'Columbia'}"],
|
||||||
|
[3, "{'address': {'city': 'New Haven', 'DT': %s, 'id': 13, 'D': %s}, " %
|
||||||
|
(repr(d2), repr(d2.date())) +
|
||||||
|
"'id': 3, 'name': 'Yale'}"],
|
||||||
|
[4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
|
||||||
|
"'id': 4, 'name': 'Yale'}"],
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_record_set(self):
|
||||||
|
self.load_sample(testsamples.sample_students)
|
||||||
|
self.add_column("Students", "schools", formula='Schools.lookupRecords(name=$schoolName)')
|
||||||
|
self.add_column("Students", "Foo", formula='repr(RECORD($schools))')
|
||||||
|
self.assertPartialData("Students", ["id", "Foo"], [
|
||||||
|
[1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," +
|
||||||
|
" {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
|
||||||
|
[2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
|
||||||
|
" {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
|
||||||
|
[3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," +
|
||||||
|
" {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
|
||||||
|
[4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
|
||||||
|
" {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
|
||||||
|
[5, "[]"],
|
||||||
|
[6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
|
||||||
|
" {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
|
||||||
|
])
|
||||||
|
|
||||||
|
# Try a field with filtered lookupRecords result, as an iterable.
|
||||||
|
self.modify_column("Students", "Foo",
|
||||||
|
formula='repr(RECORD(s for s in $schools if s.address.city.startswith("New")))')
|
||||||
|
self.assertPartialData("Students", ["id", "Foo"], [
|
||||||
|
[1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"],
|
||||||
|
[2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
|
||||||
|
[3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"],
|
||||||
|
[4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
|
||||||
|
[5, "[]"],
|
||||||
|
[6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_record_bad_calls(self):
|
||||||
|
self.load_sample(testsamples.sample_students)
|
||||||
|
self.add_column("Schools", "Foo", formula='repr(RECORD($name))')
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, objtypes.RaisedException(ValueError())],
|
||||||
|
[2, objtypes.RaisedException(ValueError())],
|
||||||
|
[3, objtypes.RaisedException(ValueError())],
|
||||||
|
[4, objtypes.RaisedException(ValueError())],
|
||||||
|
])
|
||||||
|
self.modify_column("Schools", "Foo", formula='repr(RECORD([rec] if $id == 2 else $id))')
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, objtypes.RaisedException(ValueError())],
|
||||||
|
[2, "[{'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
|
||||||
|
[3, objtypes.RaisedException(ValueError())],
|
||||||
|
[4, objtypes.RaisedException(ValueError())],
|
||||||
|
])
|
||||||
|
self.assertEqual(self.engine.get_formula_error('Schools', 'Foo', 1).error.message,
|
||||||
|
'RECORD() requires a Record or an iterable of Records')
|
||||||
|
|
||||||
|
def test_record_error_cells(self):
|
||||||
|
self.load_sample(testsamples.sample_students)
|
||||||
|
self.add_column("Schools", "Foo", formula='repr(RECORD($address))')
|
||||||
|
self.add_column("Address", "Bar", formula='$id/($id%2)')
|
||||||
|
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||||
|
[1, "{'city': 'New York', 'Bar': 11, 'id': 11}"],
|
||||||
|
[2, "{'city': 'Colombia', 'Bar': None, 'id': 12, " +
|
||||||
|
"'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"],
|
||||||
|
[3, "{'city': 'New Haven', 'Bar': 13, 'id': 13}"],
|
||||||
|
[4, "{'city': 'West Haven', 'Bar': None, 'id': 14, " +
|
||||||
|
"'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"],
|
||||||
|
])
|
||||||
Loading…
Reference in New Issue
Block a user