From 494a6833326f9d97ad653a08054e34fa2a7f107e Mon Sep 17 00:00:00 2001 From: Louis Delbosc Date: Wed, 14 Sep 2022 20:55:44 +0200 Subject: [PATCH] Export xlsx #256 (#270) XLSX export of active view / table Co-authored-by: Louis Delbosc Co-authored-by: Vincent Viers --- app/client/components/GristDoc.ts | 13 +++++- app/client/ui/ViewLayoutMenu.ts | 2 + app/common/UserAPI.ts | 14 +++---- app/server/lib/DocApi.ts | 36 +++++++++------- app/server/lib/Export.ts | 13 +++++- app/server/lib/ExportCSV.ts | 12 +----- app/server/lib/ExportXLSX.ts | 70 +++++++++++++++++++++++++++---- test/server/lib/DocApi.ts | 33 +++++++++++++++ 8 files changed, 149 insertions(+), 44 deletions(-) diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index 2ab31665..dc8e0509 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -830,6 +830,16 @@ export class GristDoc extends DisposableWithEvents { } public getCsvLink() { + const params = this._getDocApiDownloadParams(); + return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadCsvUrl(params); + } + + public getXlsxActiveViewLink() { + const params = this._getDocApiDownloadParams(); + return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadXlsxUrl(params); + } + + private _getDocApiDownloadParams() { const filters = this.viewModel.activeSection.peek().activeFilters.get().map(filterInfo => ({ colRef : filterInfo.fieldOrColumn.origCol().origColRef(), filter : filterInfo.filter() @@ -841,8 +851,7 @@ export class GristDoc extends DisposableWithEvents { activeSortSpec: JSON.stringify(this.viewModel.activeSection().activeSortSpec()), filters : JSON.stringify(filters), }; - - return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadCsvUrl(params); + return params; } public hasGranularAccessRules(): boolean { diff --git a/app/client/ui/ViewLayoutMenu.ts b/app/client/ui/ViewLayoutMenu.ts index 3640659f..692dab5a 100644 --- a/app/client/ui/ViewLayoutMenu.ts +++ b/app/client/ui/ViewLayoutMenu.ts @@ -41,6 +41,8 @@ export function makeViewLayoutMenu(viewSection: ViewSectionRec, isReadonly: bool menuItemCmd(allCommands.printSection, 'Print widget', testId('print-section')), menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}, 'Download as CSV', testId('download-section')), + menuItemLink({ href: gristDoc.getXlsxActiveViewLink(), target: '_blank', download: ''}, + 'Download as XLSX', testId('download-section')), dom.maybe((use) => ['detail', 'single'].includes(use(viewSection.parentKey)), () => menuItemCmd(allCommands.editLayout, 'Edit Card Layout', dom.cls('disabled', isReadonly))), diff --git a/app/common/UserAPI.ts b/app/common/UserAPI.ts index f8d549ad..82a7e258 100644 --- a/app/common/UserAPI.ts +++ b/app/common/UserAPI.ts @@ -346,9 +346,9 @@ export interface UserAPI { } /** - * Parameters for the download CSV endpoint (/download/csv). + * Parameters for the download CSV and XLSX endpoint (/download/csv & /download/csv). */ - export interface DownloadCsvParams { + export interface DownloadDocParams { tableId: string; viewSection?: number; activeSortSpec?: string; @@ -391,8 +391,8 @@ export interface DocAPI { // is HEAD, the result will contain a copy of any rows added or updated. compareVersion(leftHash: string, rightHash: string): Promise; getDownloadUrl(template?: boolean): string; - getDownloadXlsxUrl(): string; - getDownloadCsvUrl(params: DownloadCsvParams): string; + getDownloadXlsxUrl(params?: DownloadDocParams): string; + getDownloadCsvUrl(params: DownloadDocParams): string; /** * Exports current document to the Google Drive as a spreadsheet file. To invoke this method, first * acquire "code" via Google Auth Endpoint (see ShareMenu.ts for an example). @@ -866,11 +866,11 @@ export class DocAPIImpl extends BaseAPI implements DocAPI { return this._url + `/download?template=${Number(template)}`; } - public getDownloadXlsxUrl() { - return this._url + '/download/xlsx'; + public getDownloadXlsxUrl(params: DownloadDocParams) { + return this._url + '/download/xlsx?' + encodeQueryParams({...params}); } - public getDownloadCsvUrl(params: DownloadCsvParams) { + public getDownloadCsvUrl(params: DownloadDocParams) { // We spread `params` to work around TypeScript being overly cautious. return this._url + '/download/csv?' + encodeQueryParams({...params}); } diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index fcc82999..36de1773 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -32,9 +32,9 @@ import {DocManager} from "app/server/lib/DocManager"; import {docSessionFromRequest, makeExceptionalDocSession, OptDocSession} from "app/server/lib/DocSession"; import {DocWorker} from "app/server/lib/DocWorker"; import {IDocWorkerMap} from "app/server/lib/DocWorkerMap"; -import {parseExportParameters} from "app/server/lib/Export"; -import {downloadCSV, DownloadCSVOptions} from "app/server/lib/ExportCSV"; -import {downloadXLSX, DownloadXLSXOptions} from "app/server/lib/ExportXLSX"; +import {parseExportParameters, DownloadOptions} from "app/server/lib/Export"; +import {downloadCSV} from "app/server/lib/ExportCSV"; +import {downloadXLSX} from "app/server/lib/ExportXLSX"; import {expressWrap} from 'app/server/lib/expressWrap'; import {filterDocumentInPlace} from "app/server/lib/filterUtils"; import {googleAuthTokenMiddleware} from "app/server/lib/GoogleAuth"; @@ -736,24 +736,21 @@ export class DocWorkerApi { this._app.get('/api/docs/:docId/download/csv', canView, withDoc(async (activeDoc, req, res) => { // Query DB for doc metadata to get the doc title. const {name: docTitle} = await this._dbManager.getDoc(req); - - const params = parseExportParameters(req); - const filename = docTitle + (params.tableId === docTitle ? '' : '-' + params.tableId); - - const options: DownloadCSVOptions = { - ...params, - filename, - }; + const options = this._getDownloadOptions(req, docTitle); await downloadCSV(activeDoc, req, res, options); })); this._app.get('/api/docs/:docId/download/xlsx', canView, withDoc(async (activeDoc, req, res) => { // Query DB for doc metadata to get the doc title (to use as the filename). - const {name: filename} = await this._dbManager.getDoc(req); - - const options: DownloadXLSXOptions = {filename}; - + const {name: docTitle} = await this._dbManager.getDoc(req); + const options = !_.isEmpty(req.query) ? this._getDownloadOptions(req, docTitle) : { + filename: docTitle, + tableId: '', + viewSectionId: undefined, + filters: [], + sortOrder: [], + }; await downloadXLSX(activeDoc, req, res, options); })); @@ -815,6 +812,15 @@ export class DocWorkerApi { return docAuth.docId!; } + private _getDownloadOptions(req: Request, name: string): DownloadOptions { + const params = parseExportParameters(req); + const options: DownloadOptions = { + ...params, + filename: name + (params.tableId === name ? '' : '-' + params.tableId), + } + return options + } + private _getActiveDoc(req: RequestWithLogin): Promise { return this._docManager.fetchDoc(docSessionFromRequest(req), getDocId(req)); } diff --git a/app/server/lib/Export.ts b/app/server/lib/Export.ts index 8415192b..0f4e1a52 100644 --- a/app/server/lib/Export.ts +++ b/app/server/lib/Export.ts @@ -80,6 +80,17 @@ export interface ExportParameters { filters: Filter[]; } +/** + * Options parameters for CSV and XLSX export functions. + */ +export interface DownloadOptions { + filename: string; + tableId: string; + viewSectionId: number | undefined; + filters: Filter[]; + sortOrder: number[]; +} + interface FilteredMetaTables { [tableId: string]: TableDataAction; } @@ -97,7 +108,7 @@ export function parseExportParameters(req: express.Request): ExportParameters { tableId, viewSectionId, sortOrder, - filters + filters, }; } diff --git a/app/server/lib/ExportCSV.ts b/app/server/lib/ExportCSV.ts index 70adad72..57d7f6ba 100644 --- a/app/server/lib/ExportCSV.ts +++ b/app/server/lib/ExportCSV.ts @@ -1,21 +1,13 @@ import {ApiError} from 'app/common/ApiError'; import {createFormatter} from 'app/common/ValueFormatter'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; -import {ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export'; +import {ExportData, exportSection, exportTable, Filter, DownloadOptions} from 'app/server/lib/Export'; import log from 'app/server/lib/log'; import * as bluebird from 'bluebird'; import contentDisposition from 'content-disposition'; import csv from 'csv'; import * as express from 'express'; -export interface DownloadCSVOptions { - filename: string; - tableId: string; - viewSectionId: number | undefined; - filters: Filter[]; - sortOrder: number[]; -} - // promisify csv bluebird.promisifyAll(csv); @@ -23,7 +15,7 @@ bluebird.promisifyAll(csv); * Converts `activeDoc` to a CSV and sends the converted data through `res`. */ export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request, - res: express.Response, options: DownloadCSVOptions) { + res: express.Response, options: DownloadOptions) { log.info('Generating .csv file...'); const {filename, tableId, viewSectionId, filters, sortOrder} = options; const data = viewSectionId ? diff --git a/app/server/lib/ExportXLSX.ts b/app/server/lib/ExportXLSX.ts index 1de7ea92..f913e9d0 100644 --- a/app/server/lib/ExportXLSX.ts +++ b/app/server/lib/ExportXLSX.ts @@ -1,34 +1,86 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {createExcelFormatter} from 'app/server/lib/ExcelFormatter'; -import {ExportData, exportDoc} from 'app/server/lib/Export'; +import {ExportData, exportDoc, DownloadOptions, exportSection, exportTable, Filter} from 'app/server/lib/Export'; import {Alignment, Border, Fill, Workbook} from 'exceljs'; import * as express from 'express'; import log from 'app/server/lib/log'; import contentDisposition from 'content-disposition'; - -export interface DownloadXLSXOptions { - filename: string; -} +import { ApiError } from 'app/common/ApiError'; /** - * Converts `activeDoc` to CSV and sends the converted data through `res`. + * Converts `activeDoc` to XLSX and sends the converted data through `res`. */ export async function downloadXLSX(activeDoc: ActiveDoc, req: express.Request, - res: express.Response, {filename}: DownloadXLSXOptions) { + res: express.Response, options: DownloadOptions) { log.debug(`Generating .xlsx file`); - const data = await makeXLSX(activeDoc, req); + const {filename, tableId, viewSectionId, filters, sortOrder} = options; + // hanlding 3 cases : full XLSX export (full file), view xlsx export, table xlsx export + const data = viewSectionId ? await makeXLSXFromViewSection(activeDoc, viewSectionId, sortOrder, filters, req) + : tableId ? await makeXLSXFromTable(activeDoc, tableId, req) + : await makeXLSX(activeDoc, req); + res.set('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'); res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx')); res.send(data); log.debug('XLSX file generated'); } +/** + * Returns a XLSX stream of a view section that can be transformed or parsed. + * + * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. + * @param {Integer} viewSectionId - id of the viewsection to export. + * @param {Integer[]} activeSortOrder (optional) - overriding sort order. + * @param {Filter[]} filters (optional) - filters defined from ui. + */ + export async function makeXLSXFromViewSection( + activeDoc: ActiveDoc, + viewSectionId: number, + sortOrder: number[], + filters: Filter[], + req: express.Request, +) { + + const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, req); + const xlsx = await convertToExcel([data], req.hostname === 'localhost'); + return xlsx; +} + +/** + * Returns a XLSX stream of a table that can be transformed or parsed. + * + * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. + * @param {Integer} tableId - id of the table to export. + */ +export async function makeXLSXFromTable( + activeDoc: ActiveDoc, + tableId: string, + req: express.Request +) { + if (!activeDoc.docData) { + throw new Error('No docData in active document'); + } + + // Look up the table to make a XLSX from. + const tables = activeDoc.docData.getMetaTable('_grist_Tables'); + const tableRef = tables.findRow('tableId', tableId); + + if (tableRef === 0) { + throw new ApiError(`Table ${tableId} not found.`, 404); + } + + const data = await exportTable(activeDoc, tableRef, req); + const xlsx = await convertToExcel([data], req.hostname === 'localhost'); + return xlsx; +} + /** * Creates excel document with all tables from an active Grist document. */ export async function makeXLSX( activeDoc: ActiveDoc, - req: express.Request): Promise { + req: express.Request, +): Promise { const content = await exportDoc(activeDoc, req); const data = await convertToExcel(content, req.hostname === 'localhost'); return data; diff --git a/test/server/lib/DocApi.ts b/test/server/lib/DocApi.ts index 634fd311..dd66b79d 100644 --- a/test/server/lib/DocApi.ts +++ b/test/server/lib/DocApi.ts @@ -1804,6 +1804,39 @@ function testDocApi() { assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' }); }); + it("GET /docs/{did}/download/xlsx serves XLSX-encoded document", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/xlsx?tableId=Table1`, chimpy); + assert.equal(resp.status, 200); + assert.notEqual(resp.data, null); + }); + + it("GET /docs/{did}/download/xlsx respects permissions", async function() { + // kiwi has no access to TestDoc + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx?tableId=Table1`, kiwi); + assert.equal(resp.status, 403); + assert.deepEqual(resp.data, { error: 'No view access' }); + }); + + it("GET /docs/{did}/download/xlsx returns 404 if tableId is invalid", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx?tableId=MissingTableId`, chimpy); + assert.equal(resp.status, 404); + assert.deepEqual(resp.data, { error: 'Table MissingTableId not found.' }); + }); + + it("GET /docs/{did}/download/xlsx returns 404 if viewSectionId is invalid", async function() { + const resp = await axios.get( + `${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx?tableId=Table1&viewSection=9999`, chimpy); + assert.equal(resp.status, 404); + assert.deepEqual(resp.data, { error: 'No record 9999 in table _grist_Views_section' }); + }); + + it("GET /docs/{did}/download/xlsx returns 200 if tableId is missing", async function() { + const resp = await axios.get( + `${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx`, chimpy); + assert.equal(resp.status, 200); + assert.notEqual(resp.data, null); + }); + it('POST /workspaces/{wid}/import handles empty filenames', async function() { if (!process.env.TEST_REDIS_URL) { this.skip(); } const worker1 = await userApi.getWorkerAPI('import');