(core) Add PUT /records DocApi endpoint to AddOrUpdate records

Summary:
As designed in https://grist.quip.com/fZSrAnJKgO5j/Add-or-Update-Records-API

Current `POST /records` adds records, and `PATCH /records` updates them by row ID. This adds `PUT /records` to 'upsert' records, applying the AddOrUpdate user action. PUT was chosen because it's idempotent. Using a separate method (instead of inferring based on the request body) also cleanly separates validation, documentation, etc.

The name `require` for the new property was suggested by Paul because `where` isn't very clear when adding records.

Test Plan: New DocApi tests

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D3251

app/server/lib/DocApi.ts

@@ -56,9 +56,10 @@ const MAX_PARALLEL_REQUESTS_PER_DOC = 10;
 type WithDocHandler = (activeDoc: ActiveDoc, req: RequestWithLogin, resp: Response) => Promise<void>;
 
 // Schema validators for api endpoints that creates or updates records.
-const {RecordsPatch, RecordsPost} = t.createCheckers(DocApiTypesTI, GristDataTI);
+const {RecordsPatch, RecordsPost, RecordsPut} = t.createCheckers(DocApiTypesTI, GristDataTI);
 RecordsPatch.setReportedPath("body");
 RecordsPost.setReportedPath("body");
+RecordsPut.setReportedPath("body");
 
 /**
  * Middleware for validating request's body with a Checker instance.
@@ -265,13 +266,16 @@ export class DocWorkerApi {
       return allSame;
     }
 
+    function fieldNames(records: any[]) {
+      return new Set<string>(_.flatMap(records, r => Object.keys({...r.fields, ...r.require})));
+    }
+
     function convertToBulkColValues(records: Array<Types.Record | Types.NewRecord>): BulkColValues {
       // User might want to create empty records, without providing a field name, for example for requests:
       // { records: [{}] }; { records: [{fields:{}}] }
       // Retrieve all field names from fields property.
-      const fieldNames = new Set<string>(_.flatMap(records, r => Object.keys(r.fields ?? {})));
       const result: BulkColValues = {};
-      for (const fieldName of fieldNames) {
+      for (const fieldName of fieldNames(records)) {
         result[fieldName] = records.map(record => record.fields?.[fieldName] ?? null);
       }
       return result;
@@ -414,6 +418,31 @@ export class DocWorkerApi {
       })
     );
 
+    // Add or update records given in records format
+    this._app.put('/api/docs/:docId/tables/:tableId/records', canEdit, validate(RecordsPut),
+      withDoc(async (activeDoc, req, res) => {
+        const {records} = req.body as Types.RecordsPut;
+        const {tableId} = req.params;
+        const {noadd, noupdate, noparse, allow_empty_require} = req.query;
+        const onmany = stringParam(req.query.onmany || "first", "onmany", ["first", "none", "all"]);
+        const options = {
+          add: !isAffirmative(noadd),
+          update: !isAffirmative(noupdate),
+          on_many: onmany,
+          allow_empty_require: isAffirmative(allow_empty_require),
+        };
+        const actions = records.map(rec =>
+          ["AddOrUpdateRecord", tableId, rec.require, rec.fields || {}, options]
+        );
+        await handleSandboxError(tableId, [...fieldNames(records)], activeDoc.applyUserActions(
+          docSessionFromRequest(req),
+          actions,
+          {parseStrings: !isAffirmative(noparse)},
+        ));
+        res.json(null);
+      })
+    );
+
     // Add a new webhook and trigger
     this._app.post('/api/docs/:docId/tables/:tableId/_subscribe', isOwner,
       withDoc(async (activeDoc, req, res) => {
@@ -937,7 +966,7 @@ async function handleSandboxError<T>(tableId: string, colNames: string[], p: Pro
       if (match) {
         throw new ApiError(`Invalid row id ${match[1]}`, 400);
       }
-      match = e.message.match(/\[Sandbox\] KeyError u?'(.*?)'/);
+      match = e.message.match(/\[Sandbox] KeyError u?'(?:Table \w+ has no column )?(\w+)'/);
       if (match) {
         if (match[1] === tableId) {
           throw new ApiError(`Table not found "${tableId}"`, 404);

app/server/lib/DocApiTypes-ti.ts

@@ -17,6 +17,17 @@ export const Record = t.iface([], {
   }),
 });
 
+export const AddOrUpdateRecord = t.iface([], {
+  "require": t.intersection(t.iface([], {
+    [t.indexKey]: "CellValue",
+  }), t.iface([], {
+    "id": t.opt("number"),
+  })),
+  "fields": t.opt(t.iface([], {
+    [t.indexKey]: "CellValue",
+  })),
+});
+
 export const RecordsPatch = t.iface([], {
   "records": t.tuple("Record", t.rest(t.array("Record"))),
 });
@@ -25,10 +36,16 @@ export const RecordsPost = t.iface([], {
   "records": t.tuple("NewRecord", t.rest(t.array("NewRecord"))),
 });
 
+export const RecordsPut = t.iface([], {
+  "records": t.tuple("AddOrUpdateRecord", t.rest(t.array("AddOrUpdateRecord"))),
+});
+
 const exportedTypeSuite: t.ITypeSuite = {
   NewRecord,
   Record,
+  AddOrUpdateRecord,
   RecordsPatch,
   RecordsPost,
+  RecordsPut,
 };
 export default exportedTypeSuite;

app/server/lib/DocApiTypes.ts

@@ -15,6 +15,14 @@ export interface Record {
   fields: { [coldId: string]: CellValue };
 }
 
+/**
+ * JSON schema for api /record endpoint. Used in PUT method for adding or updating records.
+ */
+export interface AddOrUpdateRecord {
+  require: { [coldId: string]: CellValue } & { id?: number };
+  fields?: { [coldId: string]: CellValue };
+}
+
 /**
  * JSON schema for the body of api /record PATCH endpoint
  */
@@ -28,3 +36,10 @@ export interface RecordsPatch {
 export interface RecordsPost {
   records: [NewRecord, ...NewRecord[]]; // at least one record is required
 }
+
+/**
+ * JSON schema for the body of api /record PUT endpoint
+ */
+export interface RecordsPut {
+  records: [AddOrUpdateRecord, ...AddOrUpdateRecord[]]; // at least one record is required
+}

app/server/lib/FlexServer.ts

@@ -1671,7 +1671,7 @@ function allowTestLogin() {
 function trustOriginHandler(req: express.Request, res: express.Response, next: express.NextFunction) {
   if (trustOrigin(req, res)) {
     res.header("Access-Control-Allow-Credentials", "true");
-    res.header("Access-Control-Allow-Methods", "GET, PATCH, POST, DELETE, OPTIONS");
+    res.header("Access-Control-Allow-Methods", "GET, PATCH, PUT, POST, DELETE, OPTIONS");
     res.header("Access-Control-Allow-Headers", "Authorization, Content-Type, X-Requested-With");
   } else {
     throw new Error('Unrecognized origin');

app/server/lib/requestUtils.ts

@@ -218,8 +218,12 @@ export function optStringParam(p: any): string|undefined {
 }
 
 export function stringParam(p: any, name: string, allowed?: string[]): string {
-  if (typeof p !== 'string') { throw new Error(`${name} parameter should be a string: ${p}`); }
-  if (allowed && !allowed.includes(p)) { throw new Error(`${name} parameter ${p} should be one of ${allowed}`); }
+  if (typeof p !== 'string') {
+    throw new ApiError(`${name} parameter should be a string: ${p}`, 400);
+  }
+  if (allowed && !allowed.includes(p)) {
+    throw new ApiError(`${name} parameter ${p} should be one of ${allowed}`, 400);
+  }
   return p;
 }