mirror of
https://github.com/gristlabs/grist-core.git
synced 2026-03-02 04:09:24 +00:00
(core) Add BulkAddOrUpdateRecord action for efficiency
Summary: This diff adds a new `BulkAddOrUpdateRecord` user action which is what is sounds like: - A bulk version of the existing `AddOrUpdateRecord` action. - Much more efficient for operating on many records than applying many individual actions. - Column values are specified as maps from `colId` to arrays of values as usual. - Produces bulk versions of `AddRecord` and `UpdateRecord` actions instead of many individual actions. Examples of users wanting to use something like `AddOrUpdateRecord` with large numbers of records: - https://grist.slack.com/archives/C0234CPPXPA/p1651789710290879 - https://grist.slack.com/archives/C0234CPPXPA/p1660743493480119 - https://grist.slack.com/archives/C0234CPPXPA/p1660333148491559 - https://grist.slack.com/archives/C0234CPPXPA/p1663069291726159 I tested what made many `AddOrUpdateRecord` actions slow in the first place. It was almost entirely due to producing many individual `AddRecord` user actions. About half of that time was for processing the resulting `AddRecord` doc actions. Lookups and updates were not a problem. With these changes, the slowness is gone. The Python user action implementation is more complex but there are no surprises. The JS API now groups `records` based on the keys of `require` and `fields` so that `BulkAddOrUpdateRecord` can be applied to each group. Test Plan: Update and extend Python and DocApi tests. Reviewers: jarek, paulfitz Reviewed By: jarek, paulfitz Subscribers: jarek Differential Revision: https://phab.getgrist.com/D3642
This commit is contained in:
@@ -1033,8 +1033,7 @@ class TestUserActions(test_engine.EngineTestCase):
|
||||
{"color": "green"},
|
||||
{"on_many": "all"},
|
||||
[
|
||||
["UpdateRecord", "Table1", 1, {"color": "green"}],
|
||||
["UpdateRecord", "Table1", 2, {"color": "green"}],
|
||||
["BulkUpdateRecord", "Table1", [1, 2], {"color": ["green", "green"]}],
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1044,8 +1043,7 @@ class TestUserActions(test_engine.EngineTestCase):
|
||||
{"color": "greener"},
|
||||
{"on_many": "all", "allow_empty_require": True},
|
||||
[
|
||||
["UpdateRecord", "Table1", 1, {"color": "greener"}],
|
||||
["UpdateRecord", "Table1", 2, {"color": "greener"}],
|
||||
["BulkUpdateRecord", "Table1", [1, 2], {"color": ["greener", "greener"]}],
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1159,6 +1157,128 @@ class TestUserActions(test_engine.EngineTestCase):
|
||||
[["UpdateRecord", "Table1", 102, {"date": 1900800}]],
|
||||
)
|
||||
|
||||
# Empty both does nothing
|
||||
check(
|
||||
{},
|
||||
{},
|
||||
{"allow_empty_require": True},
|
||||
[],
|
||||
)
|
||||
|
||||
def test_bulk_add_or_update(self):
|
||||
sample = testutil.parse_test_sample({
|
||||
"SCHEMA": [
|
||||
[1, "Table1", [
|
||||
[1, "first_name", "Text", False, "", "first_name", ""],
|
||||
[2, "last_name", "Text", False, "", "last_name", ""],
|
||||
[4, "color", "Text", False, "", "color", ""],
|
||||
]],
|
||||
],
|
||||
"DATA": {
|
||||
"Table1": [
|
||||
["id", "first_name", "last_name"],
|
||||
[1, "John", "Doe"],
|
||||
[2, "John", "Smith"],
|
||||
],
|
||||
}
|
||||
})
|
||||
self.load_sample(sample)
|
||||
|
||||
def check(require, values, options, stored):
|
||||
self.assertPartialOutActions(
|
||||
self.apply_user_action(["BulkAddOrUpdateRecord", "Table1", require, values, options]),
|
||||
{"stored": stored},
|
||||
)
|
||||
|
||||
check(
|
||||
{
|
||||
"first_name": [
|
||||
"John",
|
||||
"John",
|
||||
"John",
|
||||
"Bob",
|
||||
],
|
||||
"last_name": [
|
||||
"Doe",
|
||||
"Smith",
|
||||
"Johnson",
|
||||
"Johnson",
|
||||
],
|
||||
},
|
||||
{
|
||||
"color": [
|
||||
"red",
|
||||
"blue",
|
||||
"green",
|
||||
"yellow",
|
||||
],
|
||||
},
|
||||
{},
|
||||
[
|
||||
["BulkAddRecord", "Table1", [3, 4], {
|
||||
"color": ["green", "yellow"],
|
||||
"first_name": ["John", "Bob"],
|
||||
"last_name": ["Johnson", "Johnson"],
|
||||
}],
|
||||
["BulkUpdateRecord", "Table1", [1, 2], {"color": ["red", "blue"]}],
|
||||
],
|
||||
)
|
||||
|
||||
with self.assertRaises(ValueError) as cm:
|
||||
check(
|
||||
{"color": ["yellow"]},
|
||||
{"color": ["red", "blue", "green"]},
|
||||
{},
|
||||
[],
|
||||
)
|
||||
self.assertEqual(
|
||||
str(cm.exception),
|
||||
'Value lists must all have the same length, '
|
||||
'got {"col_values color": 3, "require color": 1}',
|
||||
)
|
||||
|
||||
with self.assertRaises(ValueError) as cm:
|
||||
check(
|
||||
{
|
||||
"first_name": [
|
||||
"John",
|
||||
"John",
|
||||
],
|
||||
"last_name": [
|
||||
"Doe",
|
||||
],
|
||||
},
|
||||
{},
|
||||
{},
|
||||
[],
|
||||
)
|
||||
self.assertEqual(
|
||||
str(cm.exception),
|
||||
'Value lists must all have the same length, '
|
||||
'got {"require first_name": 2, "require last_name": 1}',
|
||||
)
|
||||
|
||||
with self.assertRaises(ValueError) as cm:
|
||||
check(
|
||||
{
|
||||
"first_name": [
|
||||
"John",
|
||||
"John",
|
||||
],
|
||||
"last_name": [
|
||||
"Doe",
|
||||
"Doe",
|
||||
],
|
||||
},
|
||||
{},
|
||||
{},
|
||||
[],
|
||||
)
|
||||
self.assertEqual(
|
||||
str(cm.exception),
|
||||
"require values must be unique",
|
||||
)
|
||||
|
||||
def test_reference_lookup(self):
|
||||
sample = testutil.parse_test_sample({
|
||||
"SCHEMA": [
|
||||
|
||||
Reference in New Issue
Block a user