Option to export colId as header in CSV / XSLX instead of label (#688) (#692)

app/server/lib/DocApi.ts

@@ -1202,12 +1202,13 @@ export class DocWorkerApi {
     this._app.get('/api/docs/:docId/download/xlsx', canView, withDoc(async (activeDoc, req, res) => {
       // Query DB for doc metadata to get the doc title (to use as the filename).
       const {name: docTitle} = await this._dbManager.getDoc(req);
-      const options = !_.isEmpty(req.query) ? this._getDownloadOptions(req, docTitle) : {
+      const options: DownloadOptions = !_.isEmpty(req.query) ? this._getDownloadOptions(req, docTitle) : {
         filename: docTitle,
         tableId: '',
         viewSectionId: undefined,
         filters: [],
         sortOrder: [],
+        header: 'label'
       };
       await downloadXLSX(activeDoc, req, res, options);
     }));

app/server/lib/Export.ts

@@ -17,7 +17,7 @@ import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFor
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
 import {docSessionFromRequest} from 'app/server/lib/DocSession';
-import {optIntegerParam, optJsonParam, stringParam} from 'app/server/lib/requestUtils';
+import {optIntegerParam, optJsonParam, optStringParam, stringParam} from 'app/server/lib/requestUtils';
 import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters';
 import * as express from 'express';
 import * as _ from 'underscore';
@@ -90,6 +90,8 @@ export interface ExportData {
   docSettings: DocumentSettings;
 }
 
+export type ExportHeader = 'colId' | 'label';
+
 /**
  * Export parameters that identifies a section, filters, sort order.
  */
@@ -99,6 +101,7 @@ export interface ExportParameters {
   sortOrder?: number[];
   filters?: Filter[];
   linkingFilter?: FilterColValues;
+  header?: ExportHeader;
 }
 
 /**
@@ -117,6 +120,7 @@ export function parseExportParameters(req: express.Request): ExportParameters {
   const sortOrder = optJsonParam(req.query.activeSortSpec, []) as number[];
   const filters: Filter[] = optJsonParam(req.query.filters, []);
   const linkingFilter: FilterColValues = optJsonParam(req.query.linkingFilter, null);
+  const header = optStringParam(req.query.header, 'header', {allowed: ['label', 'colId']}) as ExportHeader | undefined;
 
   return {
     tableId,
@@ -124,6 +128,7 @@ export function parseExportParameters(req: express.Request): ExportParameters {
     sortOrder,
     filters,
     linkingFilter,
+    header,
   };
 }
 

app/server/lib/ExportCSV.ts

@@ -1,7 +1,7 @@
 import {ApiError} from 'app/common/ApiError';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {FilterColValues} from "app/common/ActiveDocAPI";
-import {DownloadOptions, ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export';
+import {DownloadOptions, ExportData, ExportHeader, exportSection, exportTable, Filter} from 'app/server/lib/Export';
 import log from 'app/server/lib/log';
 import * as bluebird from 'bluebird';
 import contentDisposition from 'content-disposition';
@@ -17,11 +17,13 @@ bluebird.promisifyAll(csv);
 export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
                                   res: express.Response, options: DownloadOptions) {
   log.info('Generating .csv file...');
-  const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
+  const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, header} = options;
   const data = viewSectionId ?
-    await makeCSVFromViewSection(
-      activeDoc, viewSectionId, sortOrder || null, filters || null, linkingFilter || null, req) :
-    await makeCSVFromTable(activeDoc, tableId, req);
+    await makeCSVFromViewSection({
+      activeDoc, viewSectionId, sortOrder: sortOrder || null, filters: filters || null,
+      linkingFilter: linkingFilter || null, header, req
+    }) :
+    await makeCSVFromTable({activeDoc, tableId, header, req});
   res.set('Content-Type', 'text/csv');
   res.setHeader('Content-Disposition', contentDisposition(filename + '.csv'));
   res.send(data);
@@ -32,36 +34,51 @@ export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
  *
  * See https://github.com/wdavidw/node-csv for API details.
  *
- * @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
- * @param {Integer} viewSectionId - id of the viewsection to export.
- * @param {Integer[]} activeSortOrder (optional) - overriding sort order.
- * @param {Filter[]} filters (optional) - filters defined from ui.
+ * @param {Object} options - options for the export.
+ * @param {Object} options.activeDoc - the activeDoc that the table being converted belongs to.
+ * @param {Integer} options.viewSectionId - id of the viewsection to export.
+ * @param {Integer[]} options.activeSortOrder (optional) - overriding sort order.
+ * @param {Filter[]} options.filters (optional) - filters defined from ui.
+ * @param {FilterColValues} options.linkingFilter (optional) - linking filter defined from ui.
+ * @param {string} options.header (optional) - which field of the column to use as header
+ * @param {express.Request} options.req - the request object.
+ *
  * @return {Promise<string>} Promise for the resulting CSV.
  */
-export async function makeCSVFromViewSection(
+export async function makeCSVFromViewSection({
+  activeDoc, viewSectionId, sortOrder = null, filters = null, linkingFilter = null, header, req
+}: {
   activeDoc: ActiveDoc,
   viewSectionId: number,
   sortOrder: number[] | null,
   filters: Filter[] | null,
   linkingFilter: FilterColValues | null,
-  req: express.Request) {
+  header?: ExportHeader,
+  req: express.Request
+}) {
 
   const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, linkingFilter, req);
-  const file = convertToCsv(data);
+  const file = convertToCsv(data, { header });
   return file;
 }
 
 /**
  * Returns a csv stream of a table that can be transformed or parsed.
  *
- * @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
- * @param {Integer} tableId - id of the table to export.
+ * @param {Object} options - options for the export.
+ * @param {Object} options.activeDoc - the activeDoc that the table being converted belongs to.
+ * @param {Integer} options.tableId - id of the table to export.
+ * @param {string} options.header (optional) - which field of the column to use as header
+ * @param {express.Request} options.req - the request object.
+ *
  * @return {Promise<string>} Promise for the resulting CSV.
  */
-export async function makeCSVFromTable(
+export async function makeCSVFromTable({ activeDoc, tableId, header, req }: {
   activeDoc: ActiveDoc,
   tableId: string,
-  req: express.Request) {
+  header?: ExportHeader,
+  req: express.Request
+}) {
 
   if (!activeDoc.docData) {
     throw new Error('No docData in active document');
@@ -76,7 +93,7 @@ export async function makeCSVFromTable(
   }
 
   const data = await exportTable(activeDoc, tableRef, req);
-  const file = convertToCsv(data);
+  const file = convertToCsv(data, { header });
   return file;
 }
 
@@ -84,13 +101,13 @@ function convertToCsv({
   rowIds,
   access,
   columns: viewColumns,
-  docSettings
-}: ExportData) {
+}: ExportData, options: { header?: ExportHeader }) {
 
   // create formatters for columns
   const formatters = viewColumns.map(col => col.formatter);
   // Arrange the data into a row-indexed matrix, starting with column headers.
-  const csvMatrix = [viewColumns.map(col => col.label)];
+  const colPropertyAsHeader = options.header ?? 'label';
+  const csvMatrix = [viewColumns.map(col => col[colPropertyAsHeader])];
   // populate all the rows with values as strings
   rowIds.forEach(row => {
     csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter(row))));

app/server/lib/workerExporter.ts

@@ -1,7 +1,7 @@
 import {PassThrough} from 'stream';
 import {FilterColValues} from "app/common/ActiveDocAPI";
 import {ActiveDocSource, doExportDoc, doExportSection, doExportTable,
-        ExportData, ExportParameters, Filter} from 'app/server/lib/Export';
+        ExportData, ExportHeader, ExportParameters, Filter} from 'app/server/lib/Export';
 import {createExcelFormatter} from 'app/server/lib/ExcelFormatter';
 import * as log from 'app/server/lib/log';
 import {Alignment, Border, Buffer as ExcelBuffer, stream as ExcelWriteStream,
@@ -79,26 +79,34 @@ export async function doMakeXLSXFromOptions(
   stream: Stream,
   options: ExportParameters
 ) {
-  const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
+  const {tableId, viewSectionId, filters, sortOrder, linkingFilter, header} = options;
   if (viewSectionId) {
-    return doMakeXLSXFromViewSection(activeDocSource, testDates, stream, viewSectionId,
-      sortOrder || null, filters || null, linkingFilter || null);
+    return doMakeXLSXFromViewSection({activeDocSource, testDates, stream, viewSectionId, header,
+      sortOrder: sortOrder || null, filters: filters || null, linkingFilter: linkingFilter || null});
   } else if (tableId) {
-    return doMakeXLSXFromTable(activeDocSource, testDates, stream, tableId);
+    return doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, header});
   } else {
-    return doMakeXLSX(activeDocSource, testDates, stream);
+    return doMakeXLSX({activeDocSource, testDates, stream, header});
   }
 }
 
 /**
+ * @async
  * Returns a XLSX stream of a view section that can be transformed or parsed.
  *
- * @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
- * @param {Integer} viewSectionId - id of the viewsection to export.
- * @param {Integer[]} activeSortOrder (optional) - overriding sort order.
- * @param {Filter[]} filters (optional) - filters defined from ui.
+ * @param {Object} options - options for the export.
+ * @param {Object} options.activeDocSource - the activeDoc that the table being converted belongs to.
+ * @param {Integer} options.viewSectionId - id of the viewsection to export.
+ * @param {Integer[]} options.activeSortOrder (optional) - overriding sort order.
+ * @param {Filter[]} options.filters (optional) - filters defined from ui.
+ * @param {FilterColValues} options.linkingFilter (optional)
+ * @param {Stream} options.stream - the stream to write to.
+ * @param {boolean} options.testDates - whether to use static dates for testing.
+ * @param {string} options.header (optional) - which field of the column to use as header
  */
-async function doMakeXLSXFromViewSection(
+async function doMakeXLSXFromViewSection({
+  activeDocSource, testDates, stream, viewSectionId, sortOrder, filters, linkingFilter, header
+}: {
   activeDocSource: ActiveDocSource,
   testDates: boolean,
   stream: Stream,
@@ -106,27 +114,35 @@ async function doMakeXLSXFromViewSection(
   sortOrder: number[] | null,
   filters: Filter[] | null,
   linkingFilter: FilterColValues | null,
-) {
+  header?: ExportHeader,
+}) {
   const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters, linkingFilter);
-  const {exportTable, end} = convertToExcel(stream, testDates);
+  const {exportTable, end} = convertToExcel(stream, testDates, {header});
   exportTable(data);
   return end();
 }
 
 /**
+ * @async
  * Returns a XLSX stream of a table that can be transformed or parsed.
  *
- * @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
- * @param {Integer} tableId - id of the table to export.
+ * @param {Object} options - options for the export.
+ * @param {Object} options.activeDocSource - the activeDoc that the table being converted belongs to.
+ * @param {Integer} options.tableId - id of the table to export.
+ * @param {Stream} options.stream - the stream to write to.
+ * @param {boolean} options.testDates - whether to use static dates for testing.
+ * @param {string} options.header (optional) - which field of the column to use as header
+ *
  */
-async function doMakeXLSXFromTable(
+async function doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, header}: {
   activeDocSource: ActiveDocSource,
   testDates: boolean,
   stream: Stream,
   tableId: string,
-) {
+  header?: ExportHeader,
+}) {
   const data = await doExportTable(activeDocSource, {tableId});
-  const {exportTable, end} = convertToExcel(stream, testDates);
+  const {exportTable, end} = convertToExcel(stream, testDates, {header});
   exportTable(data);
   return end();
 }
@@ -134,12 +150,13 @@ async function doMakeXLSXFromTable(
 /**
  * Creates excel document with all tables from an active Grist document.
  */
-async function doMakeXLSX(
+async function doMakeXLSX({activeDocSource, testDates, stream, header}: {
   activeDocSource: ActiveDocSource,
   testDates: boolean,
   stream: Stream,
-): Promise<void|ExcelBuffer> {
-  const {exportTable, end} = convertToExcel(stream, testDates);
+  header?: ExportHeader,
+}): Promise<void|ExcelBuffer> {
+  const {exportTable, end} = convertToExcel(stream, testDates, {header});
   await doExportDoc(activeDocSource, async (table: ExportData) => exportTable(table));
   return end();
 }
@@ -152,7 +169,7 @@ async function doMakeXLSX(
  * (The second option is for grist-static; at the time of writing
  * WorkbookWriter doesn't appear to be available in a browser context).
  */
-function convertToExcel(stream: Stream|undefined, testDates: boolean): {
+function convertToExcel(stream: Stream|undefined, testDates: boolean, options: { header?: ExportHeader }): {
   exportTable: (table: ExportData) => void,
   end: () => Promise<void|ExcelBuffer>,
 } {
@@ -206,7 +223,8 @@ function convertToExcel(stream: Stream|undefined, testDates: boolean): {
     const formatters = columns.map(col => createExcelFormatter(col.formatter.type, col.formatter.widgetOpts));
     // Generate headers for all columns with correct styles for whole column.
     // Actual header style for a first row will be overwritten later.
-    ws.columns = columns.map((col, c) => ({ header: col.label, style: formatters[c].style() }));
+    const colHeader = options.header ?? 'label';
+    ws.columns = columns.map((col, c) => ({ header: col[colHeader], style: formatters[c].style() }));
     // style up the header row
     for (let i = 1; i <= columns.length; i++) {
       // apply to all rows (including header)

test/server/lib/DocApi.ts

@@ -230,6 +230,14 @@ describe('DocApi', function () {
 
 // Contains the tests. This is where you want to add more test.
 function testDocApi() {
+  async function generateDocAndUrl(docName: string = "Dummy") {
+    const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
+    const docId = await userApi.newDoc({name: docName}, wid);
+    const docUrl = `${serverUrl}/api/docs/${docId}`;
+    const tableUrl = `${serverUrl}/api/docs/${docId}/tables/Table1`;
+    return { docUrl, tableUrl, docId };
+  }
+
   it("creator should be owner of a created ws", async () => {
     const kiwiEmail = 'kiwi@getgrist.com';
     const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
@@ -1080,13 +1088,13 @@ function testDocApi() {
   });
 
   describe("/docs/{did}/tables/{tid}/columns", function () {
-    async function generateDocAndUrl(docName: string = "Dummy") {
-      const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
-      const docId = await userApi.newDoc({name: docName}, wid);
-      const url = `${serverUrl}/api/docs/${docId}/tables/Table1/columns`;
-      return { url, docId };
+    async function generateDocAndUrlForColumns(name: string) {
+      const { tableUrl, docId } = await generateDocAndUrl(name);
+      return {
+        docId,
+        url: `${tableUrl}/columns`,
+      };
     }
-
     describe("PUT /docs/{did}/tables/{tid}/columns", function () {
       async function getColumnFieldsMapById(url: string, params: any) {
         const result = await axios.get(url, {...chimpy, params});
@@ -1104,7 +1112,7 @@ function testDocApi() {
         expectedFieldsByColId: Record<string, object>,
         opts?: { getParams?: any }
       ) {
-        const {url} = await generateDocAndUrl('ColumnsPut');
+        const {url} = await generateDocAndUrlForColumns('ColumnsPut');
         const body: ColumnsPut = { columns };
         const resp = await axios.put(url, body, {...chimpy, params});
         assert.equal(resp.status, 200);
@@ -1175,7 +1183,7 @@ function testDocApi() {
 
       it('should forbid update by viewers', async function () {
         // given
-        const { url, docId } = await generateDocAndUrl('ColumnsPut');
+        const { url, docId } = await generateDocAndUrlForColumns('ColumnsPut');
         await userApi.updateDocPermissions(docId, {users: {'kiwi@getgrist.com': 'viewers'}});
 
         // when
@@ -1187,7 +1195,7 @@ function testDocApi() {
 
       it("should return 404 when table is not found", async function() {
         // given
-        const { url } = await generateDocAndUrl('ColumnsPut');
+        const { url } = await generateDocAndUrlForColumns('ColumnsPut');
         const notFoundUrl = url.replace("Table1", "NonExistingTable");
 
         // when
@@ -1201,7 +1209,7 @@ function testDocApi() {
 
     describe("DELETE /docs/{did}/tables/{tid}/columns/{colId}", function () {
       it('should delete some column', async function() {
-        const {url} = await generateDocAndUrl('ColumnDelete');
+        const {url} = await generateDocAndUrlForColumns('ColumnDelete');
         const deleteUrl = url + '/A';
         const resp = await axios.delete(deleteUrl, chimpy);
 
@@ -1215,7 +1223,7 @@ function testDocApi() {
       });
 
       it('should return 404 if table not found', async function() {
-        const {url} = await generateDocAndUrl('ColumnDelete');
+        const {url} = await generateDocAndUrlForColumns('ColumnDelete');
         const deleteUrl = url.replace("Table1", "NonExistingTable") + '/A';
         const resp = await axios.delete(deleteUrl, chimpy);
 
@@ -1224,7 +1232,7 @@ function testDocApi() {
       });
 
       it('should return 404 if column not found', async function() {
-        const {url} = await generateDocAndUrl('ColumnDelete');
+        const {url} = await generateDocAndUrlForColumns('ColumnDelete');
         const deleteUrl = url + '/NonExistingColId';
         const resp = await axios.delete(deleteUrl, chimpy);
 
@@ -1233,7 +1241,7 @@ function testDocApi() {
       });
 
       it('should forbid column deletion by viewers', async function() {
-        const {url, docId} = await generateDocAndUrl('ColumnDelete');
+        const {url, docId} = await generateDocAndUrlForColumns('ColumnDelete');
         await userApi.updateDocPermissions(docId, {users: {'kiwi@getgrist.com': 'viewers'}});
         const deleteUrl = url + '/A';
         const resp = await axios.delete(deleteUrl, kiwi);
@@ -2609,6 +2617,25 @@ function testDocApi() {
     assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n');
   });
 
+  it('GET /docs/{did}/download/csv with header=colId shows columns id in the header instead of their name',
+    async function () {
+      const { docUrl } = await generateDocAndUrl('csvWithColIdAsHeader');
+      const AColRef = 2;
+      const userActions = [
+        ['AddRecord', 'Table1', null, {A: 'a1', B: 'b1'}],
+        ['UpdateRecord', '_grist_Tables_column', AColRef, { untieColIdFromLabel: true }],
+        ['UpdateRecord', '_grist_Tables_column', AColRef, {
+          label: 'Column label for A',
+          colId: 'AColId'
+        }]
+      ];
+      const resp = await axios.post(`${docUrl}/apply`, userActions, chimpy);
+      assert.equal(resp.status, 200);
+      const csvResp = await axios.get(`${docUrl}/download/csv?tableId=Table1&header=colId`, chimpy);
+      assert.equal(csvResp.status, 200);
+      assert.equal(csvResp.data, 'AColId,B,C\na1,b1,\n');
+    });
+
   it("GET /docs/{did}/download/csv respects permissions", async function () {
     // kiwi has no access to TestDoc
     const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1`, kiwi);