Export xlsx #256 (#270)

XLSX export of active view / table

Co-authored-by: Louis Delbosc <louis.delbosc.prestataire@anct.gouv.fr>
Co-authored-by: Vincent Viers <vincent.viers@beta.gouv.fr>

app/client/components/GristDoc.ts

@@ -830,6 +830,16 @@ export class GristDoc extends DisposableWithEvents {
   }
 
   public getCsvLink() {
+    const params = this._getDocApiDownloadParams();
+    return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadCsvUrl(params);
+  }
+
+  public getXlsxActiveViewLink() {
+    const params = this._getDocApiDownloadParams();
+    return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadXlsxUrl(params);
+  }
+
+  private _getDocApiDownloadParams() {
     const filters = this.viewModel.activeSection.peek().activeFilters.get().map(filterInfo => ({
       colRef : filterInfo.fieldOrColumn.origCol().origColRef(),
       filter : filterInfo.filter()
@@ -841,8 +851,7 @@ export class GristDoc extends DisposableWithEvents {
       activeSortSpec: JSON.stringify(this.viewModel.activeSection().activeSortSpec()),
       filters : JSON.stringify(filters),
     };
-
+    return params;
-    return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadCsvUrl(params);
   }
 
   public hasGranularAccessRules(): boolean {

app/client/ui/ViewLayoutMenu.ts

@@ -41,6 +41,8 @@ export function makeViewLayoutMenu(viewSection: ViewSectionRec, isReadonly: bool
     menuItemCmd(allCommands.printSection, 'Print widget', testId('print-section')),
     menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''},
       'Download as CSV', testId('download-section')),
+    menuItemLink({ href: gristDoc.getXlsxActiveViewLink(), target: '_blank', download: ''},
+    'Download as XLSX', testId('download-section')),
     dom.maybe((use) => ['detail', 'single'].includes(use(viewSection.parentKey)), () =>
       menuItemCmd(allCommands.editLayout, 'Edit Card Layout',
         dom.cls('disabled', isReadonly))),

app/common/UserAPI.ts

@@ -346,9 +346,9 @@ export interface UserAPI {
 }
 
 /**
- * Parameters for the download CSV endpoint (/download/csv).
+ * Parameters for the download CSV and XLSX endpoint (/download/csv & /download/csv).
  */
- export interface DownloadCsvParams {
+ export interface DownloadDocParams {
   tableId: string;
   viewSection?: number;
   activeSortSpec?: string;
@@ -391,8 +391,8 @@ export interface DocAPI {
   // is HEAD, the result will contain a copy of any rows added or updated.
   compareVersion(leftHash: string, rightHash: string): Promise<DocStateComparison>;
   getDownloadUrl(template?: boolean): string;
-  getDownloadXlsxUrl(): string;
+  getDownloadXlsxUrl(params?: DownloadDocParams): string;
-  getDownloadCsvUrl(params: DownloadCsvParams): string;
+  getDownloadCsvUrl(params: DownloadDocParams): string;
   /**
    * Exports current document to the Google Drive as a spreadsheet file. To invoke this method, first
    * acquire "code" via Google Auth Endpoint (see ShareMenu.ts for an example).
@@ -866,11 +866,11 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
     return this._url + `/download?template=${Number(template)}`;
   }
 
-  public getDownloadXlsxUrl() {
+  public getDownloadXlsxUrl(params: DownloadDocParams) {
-    return this._url + '/download/xlsx';
+    return this._url + '/download/xlsx?' + encodeQueryParams({...params});
   }
 
-  public getDownloadCsvUrl(params: DownloadCsvParams) {
+  public getDownloadCsvUrl(params: DownloadDocParams) {
     // We spread `params` to work around TypeScript being overly cautious.
     return this._url + '/download/csv?' + encodeQueryParams({...params});
   }

app/server/lib/DocApi.ts

@@ -32,9 +32,9 @@ import {DocManager} from "app/server/lib/DocManager";
 import {docSessionFromRequest, makeExceptionalDocSession, OptDocSession} from "app/server/lib/DocSession";
 import {DocWorker} from "app/server/lib/DocWorker";
 import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
-import {parseExportParameters} from "app/server/lib/Export";
+import {parseExportParameters, DownloadOptions} from "app/server/lib/Export";
-import {downloadCSV, DownloadCSVOptions} from "app/server/lib/ExportCSV";
+import {downloadCSV} from "app/server/lib/ExportCSV";
-import {downloadXLSX, DownloadXLSXOptions} from "app/server/lib/ExportXLSX";
+import {downloadXLSX} from "app/server/lib/ExportXLSX";
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {filterDocumentInPlace} from "app/server/lib/filterUtils";
 import {googleAuthTokenMiddleware} from "app/server/lib/GoogleAuth";
@@ -736,24 +736,21 @@ export class DocWorkerApi {
     this._app.get('/api/docs/:docId/download/csv', canView, withDoc(async (activeDoc, req, res) => {
       // Query DB for doc metadata to get the doc title.
       const {name: docTitle} = await this._dbManager.getDoc(req);
-
+      const options = this._getDownloadOptions(req, docTitle);
-      const params = parseExportParameters(req);
-      const filename = docTitle + (params.tableId === docTitle ? '' : '-' + params.tableId);
-
-      const options: DownloadCSVOptions = {
-        ...params,
-        filename,
-      };
 
       await downloadCSV(activeDoc, req, res, options);
     }));
 
     this._app.get('/api/docs/:docId/download/xlsx', canView, withDoc(async (activeDoc, req, res) => {
       // Query DB for doc metadata to get the doc title (to use as the filename).
-      const {name: filename} = await this._dbManager.getDoc(req);
+      const {name: docTitle} = await this._dbManager.getDoc(req);
-
+      const options = !_.isEmpty(req.query) ? this._getDownloadOptions(req, docTitle) : {
-      const options: DownloadXLSXOptions = {filename};
+        filename: docTitle,
-
+        tableId: '',
+        viewSectionId: undefined,
+        filters: [],
+        sortOrder: [],
+      };
       await downloadXLSX(activeDoc, req, res, options);
     }));
 
@@ -815,6 +812,15 @@ export class DocWorkerApi {
     return docAuth.docId!;
   }
 
+  private _getDownloadOptions(req: Request, name: string): DownloadOptions {
+    const params = parseExportParameters(req);
+    const options: DownloadOptions = {
+      ...params,
+      filename: name + (params.tableId === name ? '' : '-' + params.tableId),
+    }
+    return options
+  }
+
   private _getActiveDoc(req: RequestWithLogin): Promise<ActiveDoc> {
     return this._docManager.fetchDoc(docSessionFromRequest(req), getDocId(req));
   }

app/server/lib/Export.ts

@@ -80,6 +80,17 @@ export interface ExportParameters {
   filters: Filter[];
 }
 
+/**
+ * Options parameters for CSV and XLSX export functions.
+ */
+export interface DownloadOptions {
+  filename: string;
+  tableId: string;
+  viewSectionId: number | undefined;
+  filters: Filter[];
+  sortOrder: number[];
+}
+
 interface FilteredMetaTables {
   [tableId: string]: TableDataAction;
 }
@@ -97,7 +108,7 @@ export function parseExportParameters(req: express.Request): ExportParameters {
     tableId,
     viewSectionId,
     sortOrder,
-    filters
+    filters,
   };
 }
 

app/server/lib/ExportCSV.ts

@@ -1,21 +1,13 @@
 import {ApiError} from 'app/common/ApiError';
 import {createFormatter} from 'app/common/ValueFormatter';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export';
+import {ExportData, exportSection, exportTable, Filter, DownloadOptions} from 'app/server/lib/Export';
 import log from 'app/server/lib/log';
 import * as bluebird from 'bluebird';
 import contentDisposition from 'content-disposition';
 import csv from 'csv';
 import * as express from 'express';
 
-export interface DownloadCSVOptions {
-  filename: string;
-  tableId: string;
-  viewSectionId: number | undefined;
-  filters: Filter[];
-  sortOrder: number[];
-}
-
 // promisify csv
 bluebird.promisifyAll(csv);
 
@@ -23,7 +15,7 @@ bluebird.promisifyAll(csv);
  * Converts `activeDoc` to a CSV and sends the converted data through `res`.
  */
 export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
-                                  res: express.Response, options: DownloadCSVOptions) {
+                                  res: express.Response, options: DownloadOptions) {
   log.info('Generating .csv file...');
   const {filename, tableId, viewSectionId, filters, sortOrder} = options;
   const data = viewSectionId ?

app/server/lib/ExportXLSX.ts

@@ -1,34 +1,86 @@
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {createExcelFormatter} from 'app/server/lib/ExcelFormatter';
-import {ExportData, exportDoc} from 'app/server/lib/Export';
+import {ExportData, exportDoc, DownloadOptions, exportSection, exportTable, Filter} from 'app/server/lib/Export';
 import {Alignment, Border, Fill, Workbook} from 'exceljs';
 import * as express from 'express';
 import log from 'app/server/lib/log';
 import contentDisposition from 'content-disposition';
-
+import { ApiError } from 'app/common/ApiError';
-export interface DownloadXLSXOptions {
-  filename: string;
-}
 
 /**
- * Converts `activeDoc` to CSV and sends the converted data through `res`.
+ * Converts `activeDoc` to XLSX and sends the converted data through `res`.
  */
 export async function downloadXLSX(activeDoc: ActiveDoc, req: express.Request,
-                                   res: express.Response, {filename}: DownloadXLSXOptions) {
+                                   res: express.Response, options: DownloadOptions) {
   log.debug(`Generating .xlsx file`);
-  const data = await makeXLSX(activeDoc, req);
+  const {filename, tableId, viewSectionId, filters, sortOrder} = options;
+  // hanlding 3 cases : full XLSX export (full file), view xlsx export, table xlsx export
+  const data = viewSectionId ? await makeXLSXFromViewSection(activeDoc, viewSectionId, sortOrder, filters, req)
+              : tableId ? await makeXLSXFromTable(activeDoc, tableId, req)
+              : await makeXLSX(activeDoc, req);
+
   res.set('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
   res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx'));
   res.send(data);
   log.debug('XLSX file generated');
 }
 
+/**
+ * Returns a XLSX stream of a view section that can be transformed or parsed.
+ *
+ * @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
+ * @param {Integer} viewSectionId - id of the viewsection to export.
+ * @param {Integer[]} activeSortOrder (optional) - overriding sort order.
+ * @param {Filter[]} filters (optional) - filters defined from ui.
+ */
+ export async function makeXLSXFromViewSection(
+  activeDoc: ActiveDoc,
+  viewSectionId: number,
+  sortOrder: number[],
+  filters: Filter[],
+  req: express.Request,
+) {
+
+  const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, req);
+  const xlsx = await convertToExcel([data], req.hostname === 'localhost');
+  return xlsx;
+}
+
+/**
+ * Returns a XLSX stream of a table that can be transformed or parsed.
+ *
+ * @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
+ * @param {Integer} tableId - id of the table to export.
+ */
+export async function makeXLSXFromTable(
+  activeDoc: ActiveDoc,
+  tableId: string,
+  req: express.Request
+) {
+  if (!activeDoc.docData) {
+    throw new Error('No docData in active document');
+  }
+
+  // Look up the table to make a XLSX from.
+  const tables = activeDoc.docData.getMetaTable('_grist_Tables');
+  const tableRef = tables.findRow('tableId', tableId);
+
+  if (tableRef === 0) {
+    throw new ApiError(`Table ${tableId} not found.`, 404);
+  }
+
+  const data = await exportTable(activeDoc, tableRef, req);
+  const xlsx = await convertToExcel([data], req.hostname === 'localhost');
+  return xlsx;
+}
+
 /**
  * Creates excel document with all tables from an active Grist document.
  */
 export async function makeXLSX(
   activeDoc: ActiveDoc,
-  req: express.Request): Promise<ArrayBuffer> {
+  req: express.Request,
+): Promise<ArrayBuffer> {
   const content = await exportDoc(activeDoc, req);
   const data = await convertToExcel(content, req.hostname === 'localhost');
   return data;

test/server/lib/DocApi.ts

@@ -1804,6 +1804,39 @@ function testDocApi() {
     assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' });
   });
 
+  it("GET /docs/{did}/download/xlsx serves XLSX-encoded document", async function() {
+    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/xlsx?tableId=Table1`, chimpy);
+    assert.equal(resp.status, 200);
+    assert.notEqual(resp.data, null);
+  });
+
+  it("GET /docs/{did}/download/xlsx respects permissions", async function() {
+    // kiwi has no access to TestDoc
+    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx?tableId=Table1`, kiwi);
+    assert.equal(resp.status, 403);
+    assert.deepEqual(resp.data, { error: 'No view access' });
+  });
+
+  it("GET /docs/{did}/download/xlsx returns 404 if tableId is invalid", async function() {
+    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx?tableId=MissingTableId`, chimpy);
+    assert.equal(resp.status, 404);
+    assert.deepEqual(resp.data, { error: 'Table MissingTableId not found.' });
+  });
+
+  it("GET /docs/{did}/download/xlsx returns 404 if viewSectionId is invalid", async function() {
+    const resp = await axios.get(
+      `${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx?tableId=Table1&viewSection=9999`, chimpy);
+    assert.equal(resp.status, 404);
+    assert.deepEqual(resp.data, { error: 'No record 9999 in table _grist_Views_section' });
+  });
+
+  it("GET /docs/{did}/download/xlsx returns 200 if tableId is missing", async function() {
+    const resp = await axios.get(
+      `${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx`, chimpy);
+    assert.equal(resp.status, 200);
+    assert.notEqual(resp.data, null);
+  });
+
   it('POST /workspaces/{wid}/import handles empty filenames', async function() {
     if (!process.env.TEST_REDIS_URL) { this.skip(); }
     const worker1 = await userApi.getWorkerAPI('import');