(core) add grist.selectedTable.create/update/destroy/upsert to custom widget api

Summary: This makes an equivalent of the /records REST endpoint available within custom widgets. For simple operations, it is compatible with https://github.com/airtable/airtable.js/. About half of the diff is refactoring code from DocApi that implements /records using applyUserActions, to make that code available in the plugin api.

Test Plan: added tests

Reviewers: alexmojaki

Reviewed By: alexmojaki

Differential Revision: https://phab.getgrist.com/D3320

app/client/components/WidgetFrame.ts

@@ -312,8 +312,8 @@ export class GristDocAPIImpl implements GristDocAPI {
     return fromTableDataAction(await this._doc.docComm.fetchTable(tableId));
   }
 
-  public async applyUserActions(actions: any[][]) {
+  public async applyUserActions(actions: any[][], options?: any) {
-    return this._doc.docComm.applyUserActions(actions, {desc: undefined});
+    return this._doc.docComm.applyUserActions(actions, {desc: undefined, ...options});
   }
 }
 

app/common/DocActions.ts

@@ -3,8 +3,8 @@
  */
 
 // Some definitions have moved to be part of plugin API.
-import { CellValue, RowRecord } from 'app/plugin/GristData';
+import { BulkColValues, CellValue, RowRecord } from 'app/plugin/GristData';
-export { CellValue, RowRecord } from 'app/plugin/GristData';
+export { BulkColValues, CellValue, RowRecord } from 'app/plugin/GristData';
 
 // Part of a special CellValue used for comparisons, embedding several versions of a CellValue.
 export interface AllCellVersions {
@@ -108,7 +108,6 @@ export function getTableId(action: DocAction): string {
 // Helper types used in the definitions above.
 
 export interface ColValues { [colId: string]: CellValue; }
-export interface BulkColValues { [colId: string]: CellValue[]; }
 export interface ColInfoMap { [colId: string]: ColInfo; }
 
 export interface ColInfo {

app/common/gutil.ts

@@ -1,9 +1,10 @@
 import {delay} from 'app/common/delay';
 import {BindableValue, DomElementMethod, ISubscribable, Listener, Observable, subscribeElem, UseCB} from 'grainjs';
 import {Observable as KoObservable} from 'knockout';
-import constant = require('lodash/constant');
 import identity = require('lodash/identity');
-import times = require('lodash/times');
+
+// Some definitions have moved to be used by plugin API.
+export {arrayRepeat} from 'app/plugin/gutil';
 
 export const UP_TRIANGLE = '\u25B2';
 export const DOWN_TRIANGLE = '\u25BC';
@@ -363,13 +364,6 @@ export function arraySplice<T>(target: T[], start: number, arrToInsert: ArrayLik
 }
 
 
-/**
- * Returns a new array of length count, filled with the given value.
- */
-export function arrayRepeat<T>(count: number, value: T): T[] {
-  return times(count, constant(value));
-}
-
 // Type for a compare func that returns a positive, negative, or zero value, as used for sorting.
 export type CompareFunc<T> = (a: T, b: T) => number;
 

app/plugin/DocApiTypes-ti.ts

@@ -40,6 +40,12 @@ export const RecordsPut = t.iface([], {
   "records": t.tuple("AddOrUpdateRecord", t.rest(t.array("AddOrUpdateRecord"))),
 });
 
+export const RecordId = t.name("number");
+
+export const MinimalRecord = t.iface([], {
+  "id": "number",
+});
+
 const exportedTypeSuite: t.ITypeSuite = {
   NewRecord,
   Record,
@@ -47,5 +53,7 @@ const exportedTypeSuite: t.ITypeSuite = {
   RecordsPatch,
   RecordsPost,
   RecordsPut,
+  RecordId,
+  MinimalRecord,
 };
 export default exportedTypeSuite;

app/plugin/DocApiTypes.ts

@@ -43,3 +43,9 @@ export interface RecordsPost {
 export interface RecordsPut {
   records: [AddOrUpdateRecord, ...AddOrUpdateRecord[]]; // at least one record is required
 }
+
+export type RecordId = number;
+
+export interface MinimalRecord {
+  id: number
+}

app/plugin/GristAPI-ti.ts

@@ -17,7 +17,7 @@ export const GristDocAPI = t.iface([], {
   "getDocName": t.func("string"),
   "listTables": t.func(t.array("string")),
   "fetchTable": t.func("any", t.param("tableId", "string")),
-  "applyUserActions": t.func("any", t.param("actions", t.array(t.array("any")))),
+  "applyUserActions": t.func("any", t.param("actions", t.array(t.array("any"))), t.param("options", "any", true)),
 });
 
 export const GristView = t.iface([], {

app/plugin/GristAPI.ts

@@ -88,7 +88,7 @@ export interface GristDocAPI {
   // Applies an array of user actions.
   // todo: return type should be Promise<ApplyUAResult>, but this requires importing modules from
   // `app/common` which is not currently supported by the build.
-  applyUserActions(actions: any[][]): Promise<any>;
+  applyUserActions(actions: any[][], options?: any): Promise<any>;
 }
 
 export interface GristView {

app/plugin/GristData-ti.ts

@@ -22,6 +22,10 @@ export const GristObjCode = t.enumtype({
 
 export const CellValue = t.union("number", "string", "boolean", "null", t.tuple("GristObjCode", t.rest(t.array("unknown"))));
 
+export const BulkColValues = t.iface([], {
+  [t.indexKey]: t.array("CellValue"),
+});
+
 export const RowRecord = t.iface([], {
   "id": "number",
   [t.indexKey]: "CellValue",
@@ -32,6 +36,7 @@ export const GristType = t.union(t.lit('Any'), t.lit('Attachments'), t.lit('Blob
 const exportedTypeSuite: t.ITypeSuite = {
   GristObjCode,
   CellValue,
+  BulkColValues,
   RowRecord,
   GristType,
 };

app/plugin/GristData.ts

@@ -16,6 +16,7 @@ export const enum GristObjCode {
 }
 
 export type CellValue = number|string|boolean|null|[GristObjCode, ...unknown[]];
+export interface BulkColValues { [colId: string]: CellValue[]; }
 
 export interface RowRecord {
   id: number;

app/plugin/TableOperations.ts

@@ -0,0 +1,44 @@
+import * as Types from 'app/plugin/DocApiTypes';
+
+/**
+ * Offer CRUD-style operations on a table.
+ */
+export interface TableOperations {
+  // Create a record or records.
+  create(records: Types.NewRecord, options?: OpOptions): Promise<Types.MinimalRecord>;
+  create(records: Types.NewRecord[], options?: OpOptions): Promise<Types.MinimalRecord[]>;
+
+  // Update a record or records.
+  update(records: Types.Record|Types.Record[], options?: OpOptions): Promise<void>;
+
+  // Delete a record or records.
+  destroy(recordId: Types.RecordId): Promise<Types.RecordId>;
+  destroy(recordIds: Types.RecordId[]): Promise<Types.RecordId[]>;
+
+  // Add or update a record or records.
+  upsert(records: Types.AddOrUpdateRecord|Types.AddOrUpdateRecord[],
+         options?: UpsertOptions): Promise<void>;
+
+  // TODO: offer a way to query the table.
+  // select(): Records;
+}
+
+/**
+ * General options for table operations.
+ * By default, string field values will be parsed based on the column type.
+ * This can be disabled.
+ */
+export interface OpOptions {
+  parseStrings?: boolean;
+}
+
+/**
+ * Extra options for upserts. By default, add and update are true,
+ * onMany is first, and allowEmptyRequire is false.
+ */
+export interface UpsertOptions extends OpOptions {
+  add?: boolean;      // permit inserting a record
+  update?: boolean;   // permit updating a record
+  onMany?: 'none' | 'first' | 'all';  // whether to update none, one, or all matching records
+  allowEmptyRequire?: boolean; // allow "wildcard" operation
+}

app/plugin/TableOperationsImpl.ts

@@ -0,0 +1,199 @@
+import * as Types from "app/plugin/DocApiTypes";
+import { BulkColValues } from 'app/plugin/GristData';
+import { OpOptions, TableOperations, UpsertOptions } from 'app/plugin/TableOperations';
+import { arrayRepeat } from './gutil';
+import flatMap = require('lodash/flatMap');
+import isEqual = require('lodash/isEqual');
+import pick = require('lodash/pick');
+
+/**
+ * An implementation of the TableOperations interface, given a platform
+ * capable of applying user actions. Used by REST API server, and by the
+ * Grist plugin API that is embedded in custom widgets.
+ */
+export class TableOperationsImpl implements TableOperations {
+  public constructor(private _platform: TableOperationsPlatform,
+                     private _defaultOptions: OpOptions) {
+  }
+
+  public create(records: Types.NewRecord, options?: OpOptions): Promise<Types.MinimalRecord>;
+  public create(records: Types.NewRecord[], options?: OpOptions): Promise<Types.MinimalRecord[]>;
+  public async create(recordsOrRecord: Types.NewRecord[]|Types.NewRecord,
+                      options?: OpOptions): Promise<Types.MinimalRecord[]|Types.MinimalRecord> {
+    return await withRecords(recordsOrRecord, async (records) => {
+      const postRecords = convertToBulkColValues(records);
+      // postRecords can be an empty object, in that case we will create empty records.
+      const ids = await this.addRecords(records.length, postRecords, options);
+      return ids.map(id => ({id}));
+    });
+  }
+
+  public async update(recordOrRecords: Types.Record|Types.Record[], options?: OpOptions) {
+    await withRecords(recordOrRecords, async (records) => {
+      if (!areSameFields(records)) {
+        this._platform.throwError('PATCH', 'requires all records to have same fields', 400);
+      }
+      const rowIds = records.map(r => r.id);
+      const columnValues = convertToBulkColValues(records);
+      if (!rowIds.length || !columnValues) {
+        // For patch method, we require at least one valid record.
+        this._platform.throwError('PATCH', 'requires a valid record object', 400);
+      }
+      await this.updateRecords(columnValues, rowIds, options);
+      return [];
+    });
+  }
+
+  public async upsert(recordOrRecords: Types.AddOrUpdateRecord|Types.AddOrUpdateRecord[],
+                      upsertOptions?: UpsertOptions): Promise<void> {
+    await withRecords(recordOrRecords, async (records) => {
+      const tableId = await this._platform.getTableId();
+      const options = {
+        add: upsertOptions?.add,
+        update: upsertOptions?.update,
+        on_many: upsertOptions?.onMany,
+        allow_empty_require: upsertOptions?.allowEmptyRequire
+      };
+      const recordOptions: OpOptions = pick(upsertOptions, 'parseStrings');
+      const actions = records.map(rec =>
+        ["AddOrUpdateRecord", tableId, rec.require, rec.fields || {}, options]);
+      await this._applyUserActions(tableId, [...fieldNames(records)],
+                                   actions, recordOptions);
+      return [];
+    });
+  }
+
+  public destroy(recordId: Types.RecordId): Promise<Types.RecordId>;
+  public destroy(recordIds: Types.RecordId[]): Promise<Types.RecordId[]>;
+  public async destroy(recordIdOrRecordIds: Types.RecordId|Types.RecordId[]): Promise<Types.RecordId|Types.RecordId[]> {
+    return withRecords(recordIdOrRecordIds, async (recordIds) => {
+      const tableId = await this._platform.getTableId();
+      const actions = [['BulkRemoveRecord', tableId, recordIds]];
+      const sandboxRes = await this._applyUserActions(
+        tableId, [], actions);
+      return sandboxRes.retValues[0];
+    });
+  }
+
+  // Update records identified by rowIds. Any invalid id fails
+  // the request and returns a 400 error code.
+  // This is exposed as a public method to support the older /data endpoint.
+  public async updateRecords(columnValues: BulkColValues, rowIds: number[],
+                             options?: OpOptions) {
+    await this._addOrUpdateRecords(columnValues, rowIds, 'BulkUpdateRecord', options);
+  }
+
+  /**
+   * Adds records to a table. If columnValues is an empty object (or not provided) it will create empty records.
+   * This is exposed as a public method to support the older /data endpoint.
+   * @param columnValues Optional values for fields (can be an empty object to add empty records)
+   * @param count Number of records to add
+   */
+  public async addRecords(
+    count: number, columnValues: BulkColValues, options?: OpOptions
+  ): Promise<number[]> {
+    // user actions expect [null, ...] as row ids
+    const rowIds = arrayRepeat(count, null);
+    return this._addOrUpdateRecords(columnValues, rowIds, 'BulkAddRecord', options);
+  }
+
+  private async _addOrUpdateRecords(
+    columnValues: BulkColValues, rowIds: (number | null)[],
+    actionType: 'BulkUpdateRecord' | 'BulkAddRecord',
+    options?: OpOptions
+  ) {
+    const tableId = await this._platform.getTableId();
+    const colNames = Object.keys(columnValues);
+    const sandboxRes = await this._applyUserActions(
+      tableId, colNames,
+      [[actionType, tableId, rowIds, columnValues]],
+      options
+    );
+    return sandboxRes.retValues[0];
+  }
+
+  // Apply the supplied actions with the given options. The tableId and
+  // colNames are just to improve error reporting.
+  private async _applyUserActions(tableId: string, colNames: string[], actions: any[][],
+                                  options: OpOptions = {}): Promise<any> {
+    return handleSandboxErrorOnPlatform(tableId, colNames, this._platform.applyUserActions(
+      actions, {...this._defaultOptions, ...options}
+    ), this._platform);
+  }
+}
+
+/**
+ * The services needed by TableOperationsImpl.
+ */
+export interface TableOperationsPlatform {
+  // Get the tableId of the table upon which we are supposed to operate.
+  getTableId(): Promise<string>;
+
+  // Throw a platform-specific error.
+  throwError(verb: string, text: string, status: number): never;
+
+  // Apply the supplied actions with the given options.
+  applyUserActions(actions: any[][], opts: any): Promise<any>;
+}
+
+export function convertToBulkColValues(records: Array<Types.Record | Types.NewRecord>): BulkColValues {
+  // User might want to create empty records, without providing a field name, for example for requests:
+  // { records: [{}] }; { records: [{fields:{}}] }
+  // Retrieve all field names from fields property.
+  const result: BulkColValues = {};
+  for (const fieldName of fieldNames(records)) {
+    result[fieldName] = records.map(record => record.fields?.[fieldName] ?? null);
+  }
+  return result;
+}
+
+export function fieldNames(records: any[]) {
+  return new Set<string>(flatMap(records, r => Object.keys({...r.fields, ...r.require})));
+}
+
+export function areSameFields(records: Array<Types.Record | Types.NewRecord>) {
+  const recordsFields = records.map(r => new Set(Object.keys(r.fields || {})));
+  return recordsFields.every(s => isEqual(recordsFields[0], s));
+}
+
+/**
+ * Adapt an operation that takes a list and returns a list to an input that may
+ * be a single object or a list. If input is empty list, return the empty list.
+ * If input is a single object, return a single object. Otherwise return a list.
+ */
+async function withRecords<T, T2>(recordsOrRecord: T[]|T, op: (records: T[]) => Promise<T2[]>): Promise<T2|T2[]> {
+  const records = Array.isArray(recordsOrRecord) ? recordsOrRecord : [recordsOrRecord];
+  const result = records.length == 0 ? [] : await op(records);
+  return Array.isArray(recordsOrRecord) ? result : result[0];
+}
+
+/**
+ * Catches the errors thrown by the sandbox, and converts to more descriptive ones (such as for
+ * invalid table names, columns, or rowIds) with better status codes. Accepts the table name, a
+ * list of column names in that table, and a promise for the result of the sandbox call.
+ */
+export async function handleSandboxErrorOnPlatform<T>(
+  tableId: string, colNames: string[], p: Promise<T>, platform: TableOperationsPlatform
+): Promise<T> {
+  try {
+    return await p;
+  } catch (err) {
+    const message = ((err instanceof Error) && err.message?.startsWith('[Sandbox] ')) ? err.message : undefined;
+    if (message) {
+      let match = message.match(/non-existent record #([0-9]+)/);
+      if (match) {
+        platform.throwError('', `Invalid row id ${match[1]}`, 400);
+      }
+      match = message.match(/\[Sandbox] KeyError u?'(?:Table \w+ has no column )?(\w+)'/);
+      if (match) {
+        if (match[1] === tableId) {
+          platform.throwError('', `Table not found "${tableId}"`, 404);
+        } else if (colNames.includes(match[1])) {
+          platform.throwError('', `Invalid column "${match[1]}"`, 400);
+        }
+      }
+      platform.throwError('', `Error manipulating data: ${message}`, 400);
+    }
+    throw err;
+  }
+}

app/plugin/grist-plugin-api.ts

@@ -25,6 +25,8 @@ import { RowRecord } from './GristData';
 import { ImportSource, ImportSourceAPI, InternalImportSourceAPI } from './InternalImportSourceAPI';
 import { decodeObject, mapValues } from './objtypes';
 import { RenderOptions, RenderTarget } from './RenderOptions';
+import { TableOperations } from './TableOperations';
+import { TableOperationsImpl } from './TableOperationsImpl';
 import { checkers } from './TypeCheckers';
 import { WidgetAPI } from './WidgetAPI';
 
@@ -68,7 +70,7 @@ export const docApi: GristDocAPI & GristView = {
     const rec = await viewApi.fetchSelectedRecord(rowId);
     return options.keepEncoded ? rec :
       mapValues(rec, decodeObject);
-  }
+  },
 };
 
 export const on = rpc.on.bind(rpc);
@@ -80,6 +82,19 @@ export const setOptions = widgetApi.setOptions.bind(widgetApi);
 export const getOptions = widgetApi.getOptions.bind(widgetApi);
 export const clearOptions = widgetApi.clearOptions.bind(widgetApi);
 
+export const selectedTable: TableOperations = new TableOperationsImpl({
+  async getTableId() {
+    await _initialization;
+    return _tableId!;
+  },
+  throwError(verb, text, status) {
+    throw new Error(text);
+  },
+  applyUserActions(actions, opts) {
+    return docApi.applyUserActions(actions, opts);
+  },
+}, {});
+
 // For custom widgets that support custom columns mappings store current configuration
 // in a memory.
 
@@ -92,6 +107,10 @@ let _mappingsCache: WidgetColumnMap|null|undefined;
 let _activeRefreshReq: Promise<void>|null = null;
 // Remember columns requested during ready call.
 let _columnsToMap: ColumnsToMap|undefined;
+let _tableId: string|undefined;
+let _setInitialized: () => void;
+const _initialization = new Promise<void>(resolve => _setInitialized = resolve);
+let _readyCalled: boolean = false;
 
 async function getMappingsIfChanged(data: any): Promise<WidgetColumnMap|null> {
   const uninitialized = _mappingsCache === undefined;
@@ -116,10 +135,12 @@ async function getMappingsIfChanged(data: any): Promise<WidgetColumnMap|null> {
  * Returns null if not all required columns were mapped or not widget doesn't support
  * custom column mapping.
  */
-export function mapColumnNames(data: any, options = {
+export function mapColumnNames(data: any, options: {
-  columns: _columnsToMap,
+  columns?: ColumnsToMap
-  mappings: _mappingsCache
+  mappings?: WidgetColumnMap|null,
+  reverse?: boolean,
 }) {
+  options = {columns: _columnsToMap, mappings: _mappingsCache, reverse: false, ...options};
   // If not column configuration was requested or
   // table has no rows, return original data.
   if (!options.columns) {
@@ -149,17 +170,30 @@ export function mapColumnNames(data: any, options = {
     );
   }
   // For each widget column in mapping.
-  for(const widgetCol in options.mappings) {
+  // Keys are ordered for determinism in case of conflicts.
+  for(const widgetCol of Object.keys(options.mappings).sort()) {
     // Get column from Grist.
     const gristCol = options.mappings[widgetCol];
     // Copy column as series (multiple values)
     if (Array.isArray(gristCol) && gristCol.length) {
+      if (!options.reverse) {
         transformations.push((from, to) => {
           to[widgetCol] = gristCol.map(col => from[col]);
         });
+      } else {
+        transformations.push((from, to) => {
+          for (const [idx, col] of gristCol.entries()) {
+            to[col] = from[widgetCol]?.[idx];
+          }
+        });
+      }
       // Copy column directly under widget column name.
     } else if (!Array.isArray(gristCol) && gristCol) {
+      if (!options.reverse) {
         transformations.push((from, to) => to[widgetCol] = from[gristCol]);
+      } else {
+        transformations.push((from, to) => to[gristCol] = from[widgetCol]);
+      }
     } else if (!isOptional(widgetCol)) {
       // Column was not configured but was required.
       return null;
@@ -171,6 +205,19 @@ export function mapColumnNames(data: any, options = {
   return Array.isArray(data) ? data.map(convert) : convert(data);
 }
 
+/**
+ * Offer a convenient way to map data with renamed columns back into the
+ * form used in the original table. This is useful for making edits to the
+ * original table in a widget with column mappings. As for mapColumnNames(),
+ * we don't attempt to do these transformations automatically.
+ */
+export function mapColumnNamesBack(data: any, options: {
+  columns?: ColumnsToMap
+  mappings?: WidgetColumnMap|null,
+}) {
+  return mapColumnNames(data, {...options, reverse: true});
+}
+
 // For custom widgets, add a handler that will be called whenever the
 // row with the cursor changes - either by switching to a different row, or
 // by some value within the row potentially changing.  Handler may
@@ -257,9 +304,19 @@ interface ReadyPayload extends Omit<InteractionOptionsRequest, "hasCustomOptions
  * Grist will not attempt to communicate with it until this method is called.
  */
 export function ready(settings?: ReadyPayload): void {
+  // Make it safe for this method to be called multiple times.
+  if (_readyCalled) { return; }
+  _readyCalled = true;
+
   if (settings && settings.onEditOptions) {
     rpc.registerFunc('editOptions', settings.onEditOptions);
   }
+  on('message', async function(msg) {
+    if (msg.tableId && msg.tableId !== _tableId) {
+      if (!_tableId) { _setInitialized(); }
+      _tableId = msg.tableId;
+    }
+  });
   rpc.processIncoming();
   void (async function() {
     await rpc.sendReadyMessage();

app/plugin/gutil.ts

@@ -0,0 +1,9 @@
+import constant = require('lodash/constant');
+import times = require('lodash/times');
+
+/**
+ * Returns a new array of length count, filled with the given value.
+ */
+export function arrayRepeat<T>(count: number, value: T): T[] {
+  return times(count, constant(value));
+}

app/server/lib/DocApi.ts

@@ -5,11 +5,14 @@ import {
   BulkColValues, ColValues, fromTableDataAction, TableColValues, TableRecordValue,
 } from 'app/common/DocActions';
 import {isRaisedException} from "app/common/gristTypes";
-import { arrayRepeat, isAffirmative } from "app/common/gutil";
+import { isAffirmative } from "app/common/gutil";
 import { SortFunc } from 'app/common/SortFunc';
 import { DocReplacementOptions, DocState, DocStateComparison, DocStates, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
 import GristDataTI from 'app/plugin/GristData-ti';
 import { HomeDBManager, makeDocAuthResult } from 'app/gen-server/lib/HomeDBManager';
+import { OpOptions } from "app/plugin/TableOperations";
+import { handleSandboxErrorOnPlatform, TableOperationsImpl,
+         TableOperationsPlatform } from 'app/plugin/TableOperationsImpl';
 import { concatenateSummaries, summarizeAction } from "app/server/lib/ActionSummary";
 import { ActiveDoc, tableIdToRef } from "app/server/lib/ActiveDoc";
 import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, getUserId, isAnonymousUser,
@@ -31,12 +34,11 @@ import { makeForkIds } from "app/server/lib/idUtils";
 import {
   getDocId, getDocScope, integerParam, isParameterOn, optStringParam,
   sendOkReply, sendReply, stringParam } from 'app/server/lib/requestUtils';
-import { SandboxError } from "app/server/lib/sandboxUtil";
 import {localeFromRequest} from "app/server/lib/ServerLocale";
 import {allowedEventTypes, isUrlAllowed, WebhookAction, WebHookSecret} from "app/server/lib/Triggers";
 import { handleOptionalUpload, handleUpload } from "app/server/lib/uploads";
-import DocApiTypesTI from "app/server/lib/DocApiTypes-ti";
+import DocApiTypesTI from "app/plugin/DocApiTypes-ti";
-import * as Types from "app/server/lib/DocApiTypes";
+import * as Types from "app/plugin/DocApiTypes";
 import * as contentDisposition from 'content-disposition';
 import { Application, NextFunction, Request, RequestHandler, Response } from "express";
 import * as _ from "lodash";
@@ -246,48 +248,14 @@ export class DocWorkerApi {
         .send(fileData);
     }));
 
-    /**
-     * Adds records to a table. If columnValues is an empty object (or not provided) it will create empty records.
-     * @param columnValues Optional values for fields (can be an empty object to add empty records)
-     * @param count Number of records to add
-     */
-    async function addRecords(
-      req: RequestWithLogin, activeDoc: ActiveDoc, count: number, columnValues: BulkColValues
-    ): Promise<number[]> {
-      // user actions expect [null, ...] as row ids
-      const rowIds = arrayRepeat(count, null);
-      return addOrUpdateRecords(req, activeDoc, columnValues, rowIds, 'BulkAddRecord');
-    }
-
-    function areSameFields(records: Array<Types.Record | Types.NewRecord>) {
-      const recordsFields = records.map(r => new Set(Object.keys(r.fields || {})));
-      const firstFields = recordsFields[0];
-      const allSame = recordsFields.every(s => _.isEqual(firstFields, s));
-      return allSame;
-    }
-
-    function fieldNames(records: any[]) {
-      return new Set<string>(_.flatMap(records, r => Object.keys({...r.fields, ...r.require})));
-    }
-
-    function convertToBulkColValues(records: Array<Types.Record | Types.NewRecord>): BulkColValues {
-      // User might want to create empty records, without providing a field name, for example for requests:
-      // { records: [{}] }; { records: [{fields:{}}] }
-      // Retrieve all field names from fields property.
-      const result: BulkColValues = {};
-      for (const fieldName of fieldNames(records)) {
-        result[fieldName] = records.map(record => record.fields?.[fieldName] ?? null);
-      }
-      return result;
-    }
-
     // Adds records given in a column oriented format,
     // returns an array of row IDs
     this._app.post('/api/docs/:docId/tables/:tableId/data', canEdit,
       withDoc(async (activeDoc, req, res) => {
         const colValues = req.body as BulkColValues;
         const count = colValues[Object.keys(colValues)[0]].length;
-        const ids = await addRecords(req, activeDoc, count, colValues);
+        const op = getTableOperations(req, activeDoc);
+        const ids = await op.addRecords(count, colValues);
         res.json(ids);
       })
     );
@@ -297,21 +265,16 @@ export class DocWorkerApi {
     this._app.post('/api/docs/:docId/tables/:tableId/records', canEdit, validate(RecordsPost),
       withDoc(async (activeDoc, req, res) => {
         const body = req.body as Types.RecordsPost;
-        const postRecords = convertToBulkColValues(body.records);
+        const ops = getTableOperations(req, activeDoc);
-        // postRecords can be an empty object, in that case we will create empty records.
+        const records = await ops.create(body.records);
-        const ids = await addRecords(req, activeDoc, body.records.length, postRecords);
-        const records = ids.map(id => ({id}));
         res.json({records});
       })
     );
 
     this._app.post('/api/docs/:docId/tables/:tableId/data/delete', canEdit, withDoc(async (activeDoc, req, res) => {
-      const tableId = req.params.tableId;
       const rowIds = req.body;
-      const sandboxRes = await handleSandboxError(tableId, [], activeDoc.applyUserActions(
+      const op = getTableOperations(req, activeDoc);
-        docSessionFromRequest(req),
+      res.json(await op.destroy(rowIds));
-        [['BulkRemoveRecord', tableId, rowIds]]));
-      res.json(sandboxRes.retValues[0]);
     }));
 
     // Download full document
@@ -364,29 +327,6 @@ export class DocWorkerApi {
       res.json({srcDocId, docId});
     }));
 
-    // Update records identified by rowIds. Any invalid id fails
-    // the request and returns a 400 error code.
-    async function updateRecords(
-      req: RequestWithLogin, activeDoc: ActiveDoc, columnValues: BulkColValues, rowIds: number[]
-    ) {
-      await addOrUpdateRecords(req, activeDoc, columnValues, rowIds, 'BulkUpdateRecord');
-    }
-
-    async function addOrUpdateRecords(
-      req: RequestWithLogin, activeDoc: ActiveDoc,
-      columnValues: BulkColValues, rowIds: (number | null)[],
-      actionType: 'BulkUpdateRecord' | 'BulkAddRecord'
-    ) {
-      const tableId = req.params.tableId;
-      const colNames = Object.keys(columnValues);
-      const sandboxRes = await handleSandboxError(tableId, colNames, activeDoc.applyUserActions(
-        docSessionFromRequest(req),
-        [[actionType, tableId, rowIds, columnValues]],
-        {parseStrings: !isAffirmative(req.query.noparse)},
-      ));
-      return sandboxRes.retValues[0];
-    }
-
     // Update records given in column format
     // The records to update are identified by their id column.
     this._app.patch('/api/docs/:docId/tables/:tableId/data', canEdit,
@@ -395,7 +335,8 @@ export class DocWorkerApi {
         const rowIds = columnValues.id;
         // sandbox expects no id column
         delete columnValues.id;
-        await updateRecords(req, activeDoc, columnValues, rowIds);
+        const ops = getTableOperations(req, activeDoc);
+        await ops.updateRecords(columnValues, rowIds);
         res.json(null);
       })
     );
@@ -404,16 +345,8 @@ export class DocWorkerApi {
     this._app.patch('/api/docs/:docId/tables/:tableId/records', canEdit, validate(RecordsPatch),
       withDoc(async (activeDoc, req, res) => {
         const body = req.body as Types.RecordsPatch;
-        const rowIds = _.map(body.records, r => r.id);
+        const ops = getTableOperations(req, activeDoc);
-        if (!areSameFields(body.records)) {
+        await ops.update(body.records);
-          throw new ApiError("PATCH requires all records to have same fields", 400);
-        }
-        const columnValues = convertToBulkColValues(body.records);
-        if (!rowIds.length || !columnValues) {
-          // For patch method, we require at least one valid record.
-          throw new ApiError("PATCH requires a valid record object", 400);
-        }
-        await updateRecords(req, activeDoc, columnValues, rowIds);
         res.json(null);
       })
     );
@@ -421,24 +354,16 @@ export class DocWorkerApi {
     // Add or update records given in records format
     this._app.put('/api/docs/:docId/tables/:tableId/records', canEdit, validate(RecordsPut),
       withDoc(async (activeDoc, req, res) => {
-        const {records} = req.body as Types.RecordsPut;
+        const ops = getTableOperations(req, activeDoc);
-        const {tableId} = req.params;
+        const body = req.body as Types.RecordsPut;
-        const {noadd, noupdate, noparse, allow_empty_require} = req.query;
-        const onmany = stringParam(req.query.onmany || "first", "onmany", ["first", "none", "all"]);
         const options = {
-          add: !isAffirmative(noadd),
+          add: !isAffirmative(req.query.noadd),
-          update: !isAffirmative(noupdate),
+          update: !isAffirmative(req.query.noupdate),
-          on_many: onmany,
+          onMany: stringParam(req.query.onmany || "first", "onmany",
-          allow_empty_require: isAffirmative(allow_empty_require),
+                              ["first", "none", "all"]) as 'first'|'none'|'all'|undefined,
+          allowEmptyRequire: isAffirmative(req.query.allow_empty_require),
         };
-        const actions = records.map(rec =>
+        await ops.upsert(body.records, options);
-          ["AddOrUpdateRecord", tableId, rec.require, rec.fields || {}, options]
-        );
-        await handleSandboxError(tableId, [...fieldNames(records)], activeDoc.applyUserActions(
-          docSessionFromRequest(req),
-          actions,
-          {parseStrings: !isAffirmative(noparse)},
-        ));
         res.json(null);
       })
     );
@@ -952,34 +877,6 @@ export function addDocApiRoutes(
   api.addEndpoints();
 }
 
-/**
- * Catches the errors thrown by the sandbox, and converts to more descriptive ones (such as for
- * invalid table names, columns, or rowIds) with better status codes. Accepts the table name, a
- * list of column names in that table, and a promise for the result of the sandbox call.
- */
-async function handleSandboxError<T>(tableId: string, colNames: string[], p: Promise<T>): Promise<T> {
-  try {
-    return await p;
-  } catch (e) {
-    if (e instanceof SandboxError) {
-      let match = e.message.match(/non-existent record #([0-9]+)/);
-      if (match) {
-        throw new ApiError(`Invalid row id ${match[1]}`, 400);
-      }
-      match = e.message.match(/\[Sandbox] KeyError u?'(?:Table \w+ has no column )?(\w+)'/);
-      if (match) {
-        if (match[1] === tableId) {
-          throw new ApiError(`Table not found "${tableId}"`, 404);
-        } else if (colNames.includes(match[1])) {
-          throw new ApiError(`Invalid column "${match[1]}"`, 400);
-        }
-      }
-      throw new ApiError(`Error doing API call: ${e.message}`, 400);
-    }
-    throw e;
-  }
-}
-
 /**
  * Options for returning results from a query about document data.
  * Currently these option don't affect the query itself, only the
@@ -1106,3 +1003,37 @@ export function applyQueryParameters(
   if (params.limit) { applyLimit(values, params.limit); }
   return values;
 }
+
+function getErrorPlatform(tableId: string): TableOperationsPlatform {
+  return {
+    async getTableId() { return tableId; },
+    throwError(verb, text, status) {
+      throw new ApiError(verb + (verb ? ' ' : '') + text, status);
+    },
+    applyUserActions() {
+      throw new Error('no document');
+    }
+  };
+}
+
+function getTableOperations(req: RequestWithLogin, activeDoc: ActiveDoc): TableOperationsImpl {
+  const options: OpOptions = {
+    parseStrings: !isAffirmative(req.query.noparse)
+  };
+  const platform: TableOperationsPlatform = {
+    ...getErrorPlatform(req.params.tableId),
+    applyUserActions(actions, opts) {
+      if (!activeDoc) { throw new Error('no document'); }
+      return activeDoc.applyUserActions(
+        docSessionFromRequest(req),
+        actions,
+        opts
+      );
+    }
+  };
+  return new TableOperationsImpl(platform, options);
+}
+
+async function handleSandboxError<T>(tableId: string, colNames: string[], p: Promise<T>): Promise<T> {
+  return handleSandboxErrorOnPlatform(tableId, colNames, p, getErrorPlatform(tableId));
+}