(core) DocApi meta endpoints: GET /tables and POST/PATCH /tables and /columns

Summary:
Adds new API endpoints to list tables in a document and create or modify tables and columns. The request and response formats are designed to mirror the style of the existing `GET /columns` and `GET/POST/PATCH /records` endpoints.

Discussion: https://grist.slack.com/archives/C0234CPPXPA/p1665139807125649?thread_ts=1628957179.010500&cid=C0234CPPXPA

Test Plan: DocApi test

Reviewers: jarek

Reviewed By: jarek

Subscribers: paulfitz

Differential Revision: https://phab.getgrist.com/D3667

app/plugin/DocApiTypes-ti.ts

@@ -10,6 +10,13 @@ export const NewRecord = t.iface([], {
   })),
 });
 
+export const NewRecordWithStringId = t.iface([], {
+  "id": t.opt("string"),
+  "fields": t.opt(t.iface([], {
+    [t.indexKey]: "CellValue",
+  })),
+});
+
 export const Record = t.iface([], {
   "id": "number",
   "fields": t.iface([], {
@@ -17,6 +24,13 @@ export const Record = t.iface([], {
   }),
 });
 
+export const RecordWithStringId = t.iface([], {
+  "id": "string",
+  "fields": t.iface([], {
+    [t.indexKey]: "CellValue",
+  }),
+});
+
 export const AddOrUpdateRecord = t.iface([], {
   "require": t.intersection(t.iface([], {
     [t.indexKey]: "CellValue",
@@ -46,14 +60,41 @@ export const MinimalRecord = t.iface([], {
   "id": "number",
 });
 
+export const ColumnsPost = t.iface([], {
+  "columns": t.tuple("NewRecordWithStringId", t.rest(t.array("NewRecordWithStringId"))),
+});
+
+export const ColumnsPatch = t.iface([], {
+  "columns": t.tuple("RecordWithStringId", t.rest(t.array("RecordWithStringId"))),
+});
+
+export const TablePost = t.iface(["ColumnsPost"], {
+  "id": t.opt("string"),
+});
+
+export const TablesPost = t.iface([], {
+  "tables": t.tuple("TablePost", t.rest(t.array("TablePost"))),
+});
+
+export const TablesPatch = t.iface([], {
+  "tables": t.tuple("RecordWithStringId", t.rest(t.array("RecordWithStringId"))),
+});
+
 const exportedTypeSuite: t.ITypeSuite = {
   NewRecord,
+  NewRecordWithStringId,
   Record,
+  RecordWithStringId,
   AddOrUpdateRecord,
   RecordsPatch,
   RecordsPost,
   RecordsPut,
   RecordId,
   MinimalRecord,
+  ColumnsPost,
+  ColumnsPatch,
+  TablePost,
+  TablesPost,
+  TablesPatch,
 };
 export default exportedTypeSuite;

app/plugin/DocApiTypes.ts

@@ -11,6 +11,15 @@ export interface NewRecord {
   fields?: { [coldId: string]: CellValue };
 }
 
+export interface NewRecordWithStringId {
+  id?: string;  // tableId or colId
+  /**
+   * Initial values of cells in record. Optional, if not set cells are left
+   * blank.
+   */
+  fields?: { [coldId: string]: CellValue };
+}
+
 /**
  * JSON schema for api /record endpoint. Used in PATCH method for updating existing records.
  */
@@ -19,6 +28,11 @@ export interface Record {
   fields: { [coldId: string]: CellValue };
 }
 
+export interface RecordWithStringId {
+  id: string;  // tableId or colId
+  fields: { [coldId: string]: CellValue };
+}
+
 /**
  * JSON schema for api /record endpoint. Used in PUT method for adding or updating records.
  */
@@ -65,3 +79,27 @@ export type RecordId = number;
 export interface MinimalRecord {
   id: number
 }
+
+export interface ColumnsPost {
+  columns: [NewRecordWithStringId, ...NewRecordWithStringId[]]; // at least one column is required
+}
+
+export interface ColumnsPatch {
+  columns: [RecordWithStringId, ...RecordWithStringId[]]; // at least one column is required
+}
+
+/**
+ * Creating tables requires a list of columns.
+ * `fields` is not accepted because it's not generally sensible to set the metadata fields on new tables.
+ */
+export interface TablePost extends ColumnsPost {
+  id?: string;
+}
+
+export interface TablesPost {
+  tables: [TablePost, ...TablePost[]]; // at least one table is required
+}
+
+export interface TablesPatch {
+  tables: [RecordWithStringId, ...RecordWithStringId[]]; // at least one table is required
+}

app/plugin/TableOperationsImpl.ts

@@ -199,7 +199,9 @@ export async function handleSandboxErrorOnPlatform<T>(
       if (match) {
         platform.throwError('', `Invalid row id ${match[1]}`, 400);
       }
-      match = message.match(/\[Sandbox] KeyError u?'(?:Table \w+ has no column )?(\w+)'/);
+      match = message.match(
+        /\[Sandbox] (?:KeyError u?'(?:Table \w+ has no column )?|ValueError No such table: )(\w+)/
+      );
       if (match) {
         if (match[1] === tableId) {
           platform.throwError('', `Table not found "${tableId}"`, 404);

app/server/lib/DocApi.ts

@@ -32,7 +32,7 @@ import {DocManager} from "app/server/lib/DocManager";
 import {docSessionFromRequest, makeExceptionalDocSession, OptDocSession} from "app/server/lib/DocSession";
 import {DocWorker} from "app/server/lib/DocWorker";
 import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
-import {parseExportParameters, DownloadOptions} from "app/server/lib/Export";
+import {DownloadOptions, parseExportParameters} from "app/server/lib/Export";
 import {downloadCSV} from "app/server/lib/ExportCSV";
 import {downloadXLSX} from "app/server/lib/ExportXLSX";
 import {expressWrap} from 'app/server/lib/expressWrap';
@@ -84,10 +84,15 @@ const MAX_ACTIVE_DOCS_USAGE_CACHE = 1000;
 type WithDocHandler = (activeDoc: ActiveDoc, req: RequestWithLogin, resp: Response) => Promise<void>;
 
 // Schema validators for api endpoints that creates or updates records.
-const {RecordsPatch, RecordsPost, RecordsPut} = t.createCheckers(DocApiTypesTI, GristDataTI);
-RecordsPatch.setReportedPath("body");
-RecordsPost.setReportedPath("body");
-RecordsPut.setReportedPath("body");
+const {
+  RecordsPatch, RecordsPost, RecordsPut,
+  ColumnsPost, ColumnsPatch,
+  TablesPost, TablesPatch,
+} = t.createCheckers(DocApiTypesTI, GristDataTI);
+
+for (const checker of [RecordsPatch, RecordsPost, RecordsPut, ColumnsPost, ColumnsPatch, TablesPost, TablesPatch]) {
+  checker.setReportedPath("body");
+}
 
 /**
  * Middleware for validating request's body with a Checker instance.
@@ -223,6 +228,21 @@ export class DocWorkerApi {
       })
     );
 
+    // Get the tables of the specified document in recordish format
+    this._app.get('/api/docs/:docId/tables', canView,
+      withDoc(async (activeDoc, req, res) => {
+        const records = await getTableRecords(activeDoc, req, "_grist_Tables");
+        const tables = records.map((record) => ({
+          id: record.fields.tableId,
+          fields: {
+            ..._.omit(record.fields, "tableId"),
+            tableRef: record.id,
+          }
+        })).filter(({id}) => id);
+        res.json({tables});
+      })
+    );
+
     // The upload should be a multipart post with an 'upload' field containing one or more files.
     // Returns the list of rowIds for the rows created in the _grist_Attachments table.
     this._app.post('/api/docs/:docId/attachments', canEdit, withDoc(async (activeDoc, req, res) => {
@@ -328,6 +348,40 @@ export class DocWorkerApi {
       })
     );
 
+    // Create columns in a table, given as records of the _grist_Tables_column metatable.
+    this._app.post('/api/docs/:docId/tables/:tableId/columns', canEdit, validate(ColumnsPost),
+      withDoc(async (activeDoc, req, res) => {
+        const body = req.body as Types.ColumnsPost;
+        const {tableId} = req.params;
+        const actions = body.columns.map(({fields, id: colId}) =>
+          // AddVisibleColumn adds the column to all widgets of the table.
+          // This isn't necessarily what the user wants, but it seems like a good default.
+          // Maybe there should be a query param to control this?
+          ["AddVisibleColumn", tableId, colId, fields || {}]
+        );
+        const {retValues} = await handleSandboxError(tableId, [],
+          activeDoc.applyUserActions(docSessionFromRequest(req), actions)
+        );
+        const columns = retValues.map(({colId}) => ({id: colId}));
+        res.json({columns});
+      })
+    );
+
+    // Create new tables in a doc. Unlike POST /records or /columns, each 'record' (table) should have a `columns`
+    // property in the same format as POST /columns above, and no `fields` property.
+    this._app.post('/api/docs/:docId/tables', canEdit, validate(TablesPost),
+      withDoc(async (activeDoc, req, res) => {
+        const body = req.body as Types.TablesPost;
+        const actions = body.tables.map(({columns, id}) => {
+          const colInfos = columns.map(({fields, id: colId}) => ({...fields, id: colId}));
+          return ["AddTable", id, colInfos];
+        });
+        const {retValues} = await activeDoc.applyUserActions(docSessionFromRequest(req), actions);
+        const tables = retValues.map(({table_id}) => ({id: table_id}));
+        res.json({tables});
+      })
+    );
+
     this._app.post('/api/docs/:docId/tables/:tableId/data/delete', canEdit, withDoc(async (activeDoc, req, res) => {
       const rowIds = req.body;
       const op = getTableOperations(req, activeDoc);
@@ -409,6 +463,48 @@ export class DocWorkerApi {
       })
     );
 
+    // Update columns given in records format
+    this._app.patch('/api/docs/:docId/tables/:tableId/columns', canEdit, validate(ColumnsPatch),
+      withDoc(async (activeDoc, req, res) => {
+        const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables");
+        const columnsTable = activeDoc.docData!.getMetaTable("_grist_Tables_column");
+        const {tableId} = req.params;
+        const tableRef = tablesTable.findMatchingRowId({tableId});
+        if (!tableRef) {
+          throw new ApiError(`Table not found "${tableId}"`, 404);
+        }
+        const body = req.body as Types.ColumnsPatch;
+        const columns: Types.Record[] = body.columns.map((col) => {
+          const id = columnsTable.findMatchingRowId({parentId: tableRef, colId: col.id});
+          if (!id) {
+            throw new ApiError(`Column not found "${col.id}"`, 404);
+          }
+          return {...col, id};
+        });
+        const ops = getTableOperations(req, activeDoc, "_grist_Tables_column");
+        await ops.update(columns);
+        res.json(null);
+      })
+    );
+
+    // Update tables given in records format
+    this._app.patch('/api/docs/:docId/tables', canEdit, validate(TablesPatch),
+      withDoc(async (activeDoc, req, res) => {
+        const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables");
+        const body = req.body as Types.TablesPatch;
+        const tables: Types.Record[] = body.tables.map((table) => {
+          const id = tablesTable.findMatchingRowId({tableId: table.id});
+          if (!id) {
+            throw new ApiError(`Table not found "${table.id}"`, 404);
+          }
+          return {...table, id};
+        });
+        const ops = getTableOperations(req, activeDoc, "_grist_Tables");
+        await ops.update(tables);
+        res.json(null);
+      })
+    );
+
     // Add or update records given in records format
     this._app.put('/api/docs/:docId/tables/:tableId/records', canEdit, validate(RecordsPut),
       withDoc(async (activeDoc, req, res) => {
@@ -1198,12 +1294,12 @@ function getErrorPlatform(tableId: string): TableOperationsPlatform {
   };
 }
 
-function getTableOperations(req: RequestWithLogin, activeDoc: ActiveDoc): TableOperationsImpl {
+function getTableOperations(req: RequestWithLogin, activeDoc: ActiveDoc, tableId?: string): TableOperationsImpl {
   const options: OpOptions = {
     parseStrings: !isAffirmative(req.query.noparse)
   };
   const platform: TableOperationsPlatform = {
-    ...getErrorPlatform(req.params.tableId),
+    ...getErrorPlatform(tableId ?? req.params.tableId),
     applyUserActions(actions, opts) {
       if (!activeDoc) { throw new Error('no document'); }
       return activeDoc.applyUserActions(