(core) Add PUT /records DocApi endpoint to AddOrUpdate records

Summary:
As designed in https://grist.quip.com/fZSrAnJKgO5j/Add-or-Update-Records-API

Current `POST /records` adds records, and `PATCH /records` updates them by row ID. This adds `PUT /records` to 'upsert' records, applying the AddOrUpdate user action. PUT was chosen because it's idempotent. Using a separate method (instead of inferring based on the request body) also cleanly separates validation, documentation, etc.

The name `require` for the new property was suggested by Paul because `where` isn't very clear when adding records.

Test Plan: New DocApi tests

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D3251

sandbox/grist/column.py

@@ -473,23 +473,8 @@ class BaseReferenceColumn(BaseColumn):
             .get_column_rec(self.table_id, self.col_id).visibleCol.colId
         or "id"
     )
-    column = self._target_table.get_column(col_id)
-    # `value` is an object encoded for transmission from JS to Python,
-    # which is decoded to `decoded_value`.
-    # `raw_value` is the kind of value that would be stored in `column`.
-    # `rich_value` is the type of value used in formulas, especially with `lookupRecords`.
-    # For example, for a Date column, `raw_value` is a numerical timestamp
-    # and `rich_value` is a `datetime.date` object,
-    # assuming `value` isn't of an invalid type.
-    # However `value` could either be just a number
-    # (in which case `decoded_value` would be a number as well)
-    # or an encoded date (or even datetime) object like ['d', number]
-    # (in which case `decoded_value` would be a `datetime.date` object,
-    #  which would get converted back to a number and then back to a date object again!)
-    decoded_value = objtypes.decode_object(value)
-    raw_value = column.convert(decoded_value)
-    rich_value = column._convert_raw_value(raw_value)
-    return self._target_table.lookup_one_record(**{col_id: rich_value})
+    value = objtypes.decode_object(value)
+    return self._target_table.lookup_one_record(**{col_id: value})
 
 
 class ReferenceColumn(BaseReferenceColumn):

sandbox/grist/table.py

@@ -443,6 +443,10 @@ class Table(object):
         # the marker is moved to col_id so that the LookupMapColumn knows how to
         # update its index correctly for that column.
         col_id = lookup._Contains(col_id)
+      else:
+        col = self.get_column(col_id)
+        # Convert `value` to the correct type of rich value for that column
+        value = col._convert_raw_value(col.convert(value))
       key.append(value)
       col_ids.append(col_id)
     col_ids = tuple(col_ids)

sandbox/grist/test_lookups.py

@@ -773,3 +773,31 @@ return ",".join(str(r.id) for r in Students.lookupRecords(firstName=fn, lastName
         ["id", "lookup"],
         [1, [None, 0, 1, 2, 3, 'foo']],
       ])
+
+  def test_conversion(self):
+    # Test that values are converted to the type of the column when looking up
+    # i.e. '123' is converted to 123
+    # and 'foo' is converted to AltText('foo')
+    self.load_sample(testutil.parse_test_sample({
+      "SCHEMA": [
+        [1, "Table1", [
+          [1, "num", "Numeric", False, "", "", ""],
+          [2, "lookup1", "RefList:Table1", True, "Table1.lookupRecords(num='123')", "", ""],
+          [3, "lookup2", "RefList:Table1", True, "Table1.lookupRecords(num='foo')", "", ""],
+        ]]
+      ],
+      "DATA": {
+        "Table1": [
+          ["id", "num"],
+          [1,    123],
+          [2,    'foo'],
+        ]
+      }
+    }))
+
+    self.assertTableData(
+      "Table1", data=[
+        ["id", "num", "lookup1", "lookup2"],
+        [1,    123,   [1],       [2]],
+        [2,    'foo', [1],       [2]],
+      ])

sandbox/grist/test_useractions.py

@@ -905,6 +905,7 @@ class TestUserActions(test_engine.EngineTestCase):
           [3, "pet",        "Text", False, "",     "pet",        ""],
           [4, "color",      "Text", False, "",     "color",      ""],
           [5, "formula",    "Text", True,  "''",   "formula",    ""],
+          [6, "date",       "Date", False, None,   "date",       ""],
         ]],
       ],
       "DATA": {
@@ -917,9 +918,9 @@ class TestUserActions(test_engine.EngineTestCase):
     })
     self.load_sample(sample)
 
-    def check(where, values, options, stored):
+    def check(require, values, options, stored):
       self.assertPartialOutActions(
-        self.apply_user_action(["AddOrUpdateRecord", "Table1", where, values, options]),
+        self.apply_user_action(["AddOrUpdateRecord", "Table1", require, values, options]),
         {"stored": stored},
       )
 
@@ -958,6 +959,26 @@ class TestUserActions(test_engine.EngineTestCase):
       ],
     )
 
+    # Update all records with empty require and allow_empty_require
+    check(
+      {},
+      {"color": "greener"},
+      {"on_many": "all", "allow_empty_require": True},
+      [
+        ["UpdateRecord", "Table1", 1, {"color": "greener"}],
+        ["UpdateRecord", "Table1", 2, {"color": "greener"}],
+      ],
+    )
+
+    # Missing allow_empty_require
+    with self.assertRaises(ValueError):
+      check(
+        {},
+        {"color": "greenest"},
+        {},
+        [],
+      )
+
     # Don't update any records when there's several matches
     check(
       {"first_name": "John"},
@@ -992,7 +1013,7 @@ class TestUserActions(test_engine.EngineTestCase):
     )
 
     # No matching record, make a new one.
-    # first_name=Jack in `values` overrides first_name=John in `where`
+    # first_name=Jack in `values` overrides first_name=John in `require`
     check(
       {"first_name": "John", "last_name": "Johnson"},
       {"first_name": "Jack", "color": "yellow"},
@@ -1003,7 +1024,7 @@ class TestUserActions(test_engine.EngineTestCase):
       ],
     )
 
-    # Specifying a row ID in `where` is allowed
+    # Specifying a row ID in `require` is allowed
     check(
       {"first_name": "Bob", "id": 100},
       {"pet": "fish"},
@@ -1019,7 +1040,7 @@ class TestUserActions(test_engine.EngineTestCase):
       [],
     )
 
-    # Nothing matches this `where`, but the row ID already exists
+    # Nothing matches this `require`, but the row ID already exists
     with self.assertRaises(AssertionError):
       check(
         {"first_name": "Alice", "id": 100},
@@ -1028,7 +1049,7 @@ class TestUserActions(test_engine.EngineTestCase):
         [],
       )
 
-    # Formula columns in `where` can't be used as values when creating records
+    # Formula columns in `require` can't be used as values when creating records
     check(
       {"formula": "anything"},
       {"first_name": "Alice"},
@@ -1036,6 +1057,29 @@ class TestUserActions(test_engine.EngineTestCase):
       [["AddRecord", "Table1", 101, {"first_name": "Alice"}]],
     )
 
+    with self.assertRaises(ValueError):
+      # Row ID too high
+      check(
+        {"first_name": "Alice", "id": 2000000},
+        {"pet": "fish"},
+        {},
+        [],
+      )
+
+    # Check that encoded objects are decoded correctly
+    check(
+      {"date": ['d', 950400]},
+      {},
+      {},
+      [["AddRecord", "Table1", 102, {"date": 950400}]],
+    )
+    check(
+      {"date": ['d', 950400]},
+      {"date": ['d', 1900800]},
+      {},
+      [["UpdateRecord", "Table1", 102, {"date": 1900800}]],
+    )
+
   def test_reference_lookup(self):
     sample = testutil.parse_test_sample({
       "SCHEMA": [

sandbox/grist/useractions.py

@@ -13,7 +13,7 @@ import actions
 import column
 import sort_specs
 import identifiers
-from objtypes import strict_equal, encode_object
+from objtypes import strict_equal, encode_object, decode_object
 import schema
 from schema import RecalcWhen
 import summary
@@ -319,6 +319,8 @@ class UserActions(object):
     for i, row_id in enumerate(filled_row_ids):
       if row_id is None or row_id < 0:
         filled_row_ids[i] = row_id = next_row_id
+      elif row_id > 1000000:
+        raise ValueError("Row ID too high")
       next_row_id = max(next_row_id, row_id) + 1
 
     # Whenever we add new rows, remember the mapping from any negative row_ids to their final
@@ -793,9 +795,35 @@ class UserActions(object):
       raise ValueError("Can't save value to formula column %s" % col_id)
 
   @useraction
-  def AddOrUpdateRecord(self, table_id, where, col_values, options):
+  def AddOrUpdateRecord(self, table_id, require, col_values, options):
+    """
+    Add or Update ('upsert') a single record depending on `options`
+    and on whether a record matching `require` already exists.
+
+    `require` and `col_values` are dictionaries mapping column IDs to single cell values.
+
+    By default, if `table.lookupRecords(**require)` returns any records,
+    update the first one with the values in `col_values`.
+    Otherwise create a new record with values `{**require, **col_values}`.
+
+    `options` is a dictionary with optional settings to choose other behaviours:
+    - Set "on_many" to "all" or "none" to change which records are updated when several match.
+    - Set "update" or "add" to False to disable updating or adding records respectively,
+      i.e. if you only want to add records that don't already exist
+        or if you only want to update records that do already exist.
+    - Set "allow_empty_require" to True to allow `require` to be an empty dictionary,
+      which would mean that every record in the table is matched.
+      Otherwise this will raise an error to prevent mistakes like updating an entire column.
+    """
     table = self._engine.tables[table_id]
-    records = list(table.lookup_records(**where))
+
+    if not require and not options.get("allow_empty_require", False):
+      raise ValueError("require is empty but allow_empty_require isn't set")
+
+    # Decode `require` before looking up, but let AddRecord/UpdateRecord decode the final
+    # values when adding/updating
+    decoded_require = {k: decode_object(v) for k, v in six.iteritems(require)}
+    records = list(table.lookup_records(**decoded_require))
 
     if records and options.get("update", True):
       if len(records) > 1:
@@ -813,8 +841,12 @@ class UserActions(object):
     if not records and options.get("add", True):
       values = {
         key: value
-        for key, value in six.iteritems(where)
-        if not table.get_column(key).is_formula()
+        for key, value in six.iteritems(require)
+        if not (
+          table.get_column(key).is_formula() and
+          # Check that there actually is a formula and this isn't just an empty column
+          self._engine.docmodel.get_column_rec(table_id, key).formula
+        )
       }
       values.update(col_values)
       self.AddRecord(table_id, values.pop("id", None), values)