DocApi: Introduce hidden option for GET on records (#623)

Also test that PUT /columns?replaceall=1 does not remove hidden columns

app/server/lib/DocApi.ts

@@ -201,16 +201,23 @@ export class DocWorkerApi {
     }
 
     async function getTableRecords(
-      activeDoc: ActiveDoc, req: RequestWithLogin, optTableId?: string
+      activeDoc: ActiveDoc, req: RequestWithLogin, opts?: { optTableId?: string; includeHidden?: boolean }
     ): Promise<TableRecordValue[]> {
-      const columnData = await getTableData(activeDoc, req, optTableId);
+      const columnData = await getTableData(activeDoc, req, opts?.optTableId);
-      const fieldNames = Object.keys(columnData)
+      const fieldNames = Object.keys(columnData).filter((k) => {
-        .filter(k => !(
+        if (k === "id") {
-          ["id", "manualSort"].includes(k)
+          return false;
-          || k.startsWith("gristHelper_")
+        }
-        ));
+        if (
+          !opts?.includeHidden &&
+          (k === "manualSort" || k.startsWith("gristHelper_"))
+        ) {
+          return false;
+        }
+        return true;
+      });
       return columnData.id.map((id, index) => {
-        const result: TableRecordValue = {id, fields: {}};
+        const result: TableRecordValue = { id, fields: {} };
         for (const key of fieldNames) {
           let value = columnData[key][index];
           if (isRaisedException(value)) {
@@ -233,7 +240,9 @@ export class DocWorkerApi {
     // Get the specified table in record-oriented format
     this._app.get('/api/docs/:docId/tables/:tableId/records', canView,
       withDoc(async (activeDoc, req, res) => {
-        const records = await getTableRecords(activeDoc, req);
+        const records = await getTableRecords(activeDoc, req,
+          { includeHidden: isAffirmative(req.query.hidden) }
+        );
         res.json({records});
       })
     );
@@ -364,7 +373,7 @@ export class DocWorkerApi {
     this._app.get('/api/docs/:docId/tables/:tableId/columns', canView,
       withDoc(async (activeDoc, req, res) => {
         const tableId = req.params.tableId;
-        const includeHidden = isAffirmative(req.query.includeHidden);
+        const includeHidden = isAffirmative(req.query.hidden);
         const columns = await handleSandboxError('', [],
           activeDoc.getTableCols(docSessionFromRequest(req), tableId, includeHidden));
         res.json({columns});
@@ -374,7 +383,7 @@ export class DocWorkerApi {
     // Get the tables of the specified document in recordish format
     this._app.get('/api/docs/:docId/tables', canView,
       withDoc(async (activeDoc, req, res) => {
-        const records = await getTableRecords(activeDoc, req, "_grist_Tables");
+        const records = await getTableRecords(activeDoc, req, { optTableId: "_grist_Tables" });
         const tables = records.map((record) => ({
           id: record.fields.tableId,
           fields: {
@@ -403,7 +412,7 @@ export class DocWorkerApi {
 
     // Returns cleaned metadata for all attachments in /records format.
     this._app.get('/api/docs/:docId/attachments', canView, withDoc(async (activeDoc, req, res) => {
-      const rawRecords = await getTableRecords(activeDoc, req, "_grist_Attachments");
+      const rawRecords = await getTableRecords(activeDoc, req, { optTableId: "_grist_Attachments" });
       const records = rawRecords.map(r => ({
         id: r.id,
         fields: cleanAttachmentRecord(r.fields as MetaRowRecord<"_grist_Attachments">),

test/server/lib/DocApi.ts

@@ -443,6 +443,26 @@ function testDocApi() {
       });
   });
 
+  it('GET /docs/{did}/tables/{tid}/records honors the "hidden" param', async function () {
+    const params = { hidden: true };
+    const resp = await axios.get(
+      `${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/records`,
+      {...chimpy, params }
+    );
+    assert.equal(resp.status, 200);
+    assert.deepEqual(resp.data.records[0], {
+      id: 1,
+      fields: {
+        manualSort: 1,
+        A: 'hello',
+        B: '',
+        C: '',
+        D: null,
+        E: 'HELLO',
+      },
+    });
+  });
+
   it("GET /docs/{did}/tables/{tid}/records handles errors and hidden columns", async function () {
     let resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/records`, chimpy);
     assert.equal(resp.status, 200);
@@ -610,8 +630,8 @@ function testDocApi() {
     );
   });
 
-  it("GET /docs/{did}/tables/{tid}/columns retrieves hidden columns when includeHidden is set", async function () {
+  it('GET /docs/{did}/tables/{tid}/columns retrieves hidden columns when "hidden" is set', async function () {
-    const params = { includeHidden: true };
+    const params = { hidden: true };
     const resp = await axios.get(
       `${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/columns`,
       { ...chimpy, params }
@@ -866,8 +886,8 @@ function testDocApi() {
       return { url, docId };
     }
 
-    async function getColumnFieldsMapById(url: string) {
+    async function getColumnFieldsMapById(url: string, params: any) {
-      const result = await axios.get(url, chimpy);
+      const result = await axios.get(url, {...chimpy, params});
       assert.equal(result.status, 200);
       return new Map<string, object>(
           result.data.columns.map(
@@ -880,12 +900,13 @@ function testDocApi() {
       columns: [RecordWithStringId, ...RecordWithStringId[]],
       params: Record<string, any>,
       expectedFieldsByColId: Record<string, object>,
+      opts?: { getParams?: any }
     ) {
       const {url} = await generateDocAndUrl();
       const body: ColumnsPut = { columns };
       const resp = await axios.put(url, body, {...chimpy, params});
       assert.equal(resp.status, 200);
-      const fieldsByColId = await getColumnFieldsMapById(url);
+      const fieldsByColId = await getColumnFieldsMapById(url, opts?.getParams);
 
       assert.deepEqual(
         [...fieldsByColId.keys()],
@@ -944,6 +965,12 @@ function testDocApi() {
       });
     });
 
+    it('should NOT remove hidden columns even when replaceall is set', async function () {
+      await checkPut([COLUMN_TO_ADD, COLUMN_TO_UPDATE], {replaceall: "1"}, {
+        manualSort: {type: "ManualSortPos"}, NewA: {type: "Numeric"}, Foo: COLUMN_TO_ADD.fields
+      }, { getParams: { hidden: true } });
+    });
+
     it('should forbid update by viewers', async function () {
       // given
       const { url, docId } = await generateDocAndUrl();