diff --git a/app/client/DefaultHooks.ts b/app/client/DefaultHooks.ts index 216d80d3..a35d7559 100644 --- a/app/client/DefaultHooks.ts +++ b/app/client/DefaultHooks.ts @@ -1,11 +1,22 @@ import { UrlTweaks } from 'app/common/gristUrls'; +import { IAttrObj } from 'grainjs'; export interface IHooks { iframeAttributes?: Record, fetch?: typeof fetch, baseURI?: string, urlTweaks?: UrlTweaks, + + /** + * Modify the attributes of an dom element. + * Convenient in grist-static to directly hook up a + * download link with the function that provides the data. + */ + maybeModifyLinkAttrs(attrs: IAttrObj): IAttrObj; } export const defaultHooks: IHooks = { + maybeModifyLinkAttrs(attrs: IAttrObj) { + return attrs; + } }; diff --git a/app/client/ui/MakeCopyMenu.ts b/app/client/ui/MakeCopyMenu.ts index a23a4636..ec3cbe16 100644 --- a/app/client/ui/MakeCopyMenu.ts +++ b/app/client/ui/MakeCopyMenu.ts @@ -3,6 +3,7 @@ * the sample documents (those in the Support user's Examples & Templates workspace). */ +import {hooks} from 'app/client/Hooks'; import {makeT} from 'app/client/lib/localization'; import {AppModel, reportError} from 'app/client/models/AppModel'; import {DocPageModel} from "app/client/models/DocPageModel"; @@ -302,14 +303,14 @@ export function downloadDocModal(doc: Document, pageModel: DocPageModel) { ), cssModalButtons( dom.domComputed(use => - bigPrimaryButtonLink(`Download`, { + bigPrimaryButtonLink(`Download`, hooks.maybeModifyLinkAttrs({ href: pageModel.appModel.api.getDocAPI(doc.id).getDownloadUrl({ template: use(selected) === "template", removeHistory: use(selected) === "nohistory" || use(selected) === "template", }), target: '_blank', download: '' - }, + }), dom.on('click', () => { ctl.close(); }), diff --git a/app/client/ui/ShareMenu.ts b/app/client/ui/ShareMenu.ts index 97a24491..4cdf37e4 100644 --- a/app/client/ui/ShareMenu.ts +++ b/app/client/ui/ShareMenu.ts @@ -1,3 +1,4 @@ +import {hooks} from 'app/client/Hooks'; import {loadUserManager} from 'app/client/lib/imports'; import {AppModel, reportError} from 'app/client/models/AppModel'; import {DocInfo, DocPageModel} from 'app/client/models/DocPageModel'; @@ -255,12 +256,12 @@ function menuExports(doc: Document, pageModel: DocPageModel) { menuItem(() => downloadDocModal(doc, pageModel), menuIcon('Download'), t("Download..."), testId('tb-share-option')) ), - menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}, + menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}), menuIcon('Download'), t("Export CSV"), testId('tb-share-option')), - menuItemLink({ + menuItemLink(hooks.maybeModifyLinkAttrs({ href: pageModel.appModel.api.getDocAPI(doc.id).getDownloadXlsxUrl(), target: '_blank', download: '' - }, menuIcon('Download'), t("Export XLSX"), testId('tb-share-option')), + }), menuIcon('Download'), t("Export XLSX"), testId('tb-share-option')), (!isFeatureEnabled("sendToDrive") ? null : menuItem(() => sendToDrive(doc, pageModel), menuIcon('Download'), t("Send to Google Drive"), testId('tb-share-option'))), ]; diff --git a/app/client/ui/ViewLayoutMenu.ts b/app/client/ui/ViewLayoutMenu.ts index 4e60490b..8a39ad7f 100644 --- a/app/client/ui/ViewLayoutMenu.ts +++ b/app/client/ui/ViewLayoutMenu.ts @@ -1,3 +1,4 @@ +import {hooks} from 'app/client/Hooks'; import {makeT} from 'app/client/lib/localization'; import {allCommands} from 'app/client/components/commands'; import {ViewSectionRec} from 'app/client/models/DocModel'; @@ -76,9 +77,9 @@ export function makeViewLayoutMenu(viewSection: ViewSectionRec, isReadonly: bool ) ), menuItemCmd(allCommands.printSection, t("Print widget"), testId('print-section')), - menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}, + menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}), t("Download as CSV"), testId('download-section')), - menuItemLink({ href: gristDoc.getXlsxActiveViewLink(), target: '_blank', download: ''}, + menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getXlsxActiveViewLink(), target: '_blank', download: ''}), t("Download as XLSX"), testId('download-section')), dom.maybe((use) => ['detail', 'single'].includes(use(viewSection.parentKey)), () => menuItemCmd(allCommands.editLayout, t("Edit Card Layout"), diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index 3533cbf2..a26fbcbf 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -49,7 +49,7 @@ import {IDocWorkerMap} from "app/server/lib/DocWorkerMap"; import {DownloadOptions, parseExportParameters} from "app/server/lib/Export"; import {downloadCSV} from "app/server/lib/ExportCSV"; import {collectTableSchemaInFrictionlessFormat} from "app/server/lib/ExportTableSchema"; -import {downloadXLSX} from "app/server/lib/ExportXLSX"; +import {streamXLSX} from "app/server/lib/ExportXLSX"; import {expressWrap} from 'app/server/lib/expressWrap'; import {filterDocumentInPlace} from "app/server/lib/filterUtils"; import {googleAuthTokenMiddleware} from "app/server/lib/GoogleAuth"; @@ -1911,3 +1911,14 @@ export interface WebhookSubscription { unsubscribeKey: string; webhookId: string; } + +/** + * Converts `activeDoc` to XLSX and sends the converted data through `res`. + */ +export async function downloadXLSX(activeDoc: ActiveDoc, req: Request, + res: Response, options: DownloadOptions) { + const {filename} = options; + res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'); + res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx')); + return streamXLSX(activeDoc, req, res, options); +} diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts index c2b5768f..5eaf2169 100644 --- a/app/server/lib/DocManager.ts +++ b/app/server/lib/DocManager.ts @@ -133,6 +133,14 @@ export class DocManager extends EventEmitter { return this.createNamedDoc(docSession, 'Untitled'); } + /** + * Add an ActiveDoc created externally. This is a hook used by + * grist-static. + */ + public addActiveDoc(docId: string, activeDoc: ActiveDoc) { + this._activeDocs.set(docId, Promise.resolve(activeDoc)); + } + public async createNamedDoc(docSession: OptDocSession, docId: string): Promise { const activeDoc: ActiveDoc = await this.createNewEmptyDoc(docSession, docId); await activeDoc.addInitialTable(docSession); diff --git a/app/server/lib/ExportXLSX.ts b/app/server/lib/ExportXLSX.ts index b18d828c..99d9b5af 100644 --- a/app/server/lib/ExportXLSX.ts +++ b/app/server/lib/ExportXLSX.ts @@ -1,8 +1,7 @@ /** * Overview of Excel exports, which now use worker-threads. * - * 1. The flow starts with downloadXLSX() method called in the main thread (or streamXLSX() used for - * Google Drive export). + * 1. The flow starts with the streamXLSX() method called in the main thread. * 2. It uses the 'piscina' library to call a makeXLSX* method in a worker thread, registered in * workerExporter.ts, to export full doc, a table, or a section. * 3. Each of those methods calls a doMakeXLSX* method defined in that file. I.e. downloadXLSX() @@ -12,11 +11,10 @@ * 5. The resulting stream of Excel data is streamed back to the main thread using Rpc too. */ import {ActiveDoc} from 'app/server/lib/ActiveDoc'; -import {ActiveDocSource, ActiveDocSourceDirect, DownloadOptions, ExportParameters} from 'app/server/lib/Export'; +import {ActiveDocSource, ActiveDocSourceDirect, ExportParameters} from 'app/server/lib/Export'; import log from 'app/server/lib/log'; import {addAbortHandler} from 'app/server/lib/requestUtils'; import * as express from 'express'; -import contentDisposition from 'content-disposition'; import {Rpc} from 'grain-rpc'; import {AbortController} from 'node-abort-controller'; import {Writable} from 'stream'; @@ -38,24 +36,12 @@ const exportPool = new Piscina({ idleTimeout: 10_000, // Drop unused threads after 10s of inactivity. }); -/** - * Converts `activeDoc` to XLSX and sends the converted data through `res`. - */ -export async function downloadXLSX(activeDoc: ActiveDoc, req: express.Request, - res: express.Response, options: DownloadOptions) { - const {filename} = options; - res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'); - res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx')); - return streamXLSX(activeDoc, req, res, options); -} - /** * Converts `activeDoc` to XLSX and sends to the given outputStream. */ export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request, outputStream: Writable, options: ExportParameters) { log.debug(`Generating .xlsx file`); - const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options; const testDates = (req.hostname === 'localhost'); const { port1, port2 } = new MessageChannel(); @@ -89,13 +75,7 @@ export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request, // hanlding 3 cases : full XLSX export (full file), view xlsx export, table xlsx export try { - if (viewSectionId) { - await run('makeXLSXFromViewSection', viewSectionId, sortOrder, filters, linkingFilter); - } else if (tableId) { - await run('makeXLSXFromTable', tableId); - } else { - await run('makeXLSX'); - } + await run('makeXLSXFromOptions', options); log.debug('XLSX file generated'); } catch (e) { // We fiddle with errors in workerExporter to preserve extra properties like 'status'. Make diff --git a/app/server/lib/workerExporter.ts b/app/server/lib/workerExporter.ts index bb8f6c12..c6992c20 100644 --- a/app/server/lib/workerExporter.ts +++ b/app/server/lib/workerExporter.ts @@ -1,19 +1,19 @@ import {PassThrough} from 'stream'; import {FilterColValues} from "app/common/ActiveDocAPI"; -import {ActiveDocSource, doExportDoc, doExportSection, doExportTable, ExportData, Filter} from 'app/server/lib/Export'; +import {ActiveDocSource, doExportDoc, doExportSection, doExportTable, + ExportData, ExportParameters, Filter} from 'app/server/lib/Export'; import {createExcelFormatter} from 'app/server/lib/ExcelFormatter'; import * as log from 'app/server/lib/log'; -import {Alignment, Border, stream as ExcelWriteStream, Fill} from 'exceljs'; +import {Alignment, Border, Buffer as ExcelBuffer, stream as ExcelWriteStream, + Fill, Workbook} from 'exceljs'; import {Rpc} from 'grain-rpc'; import {Stream} from 'stream'; import {MessagePort, threadId} from 'worker_threads'; -export const makeXLSX = handleExport(doMakeXLSX); -export const makeXLSXFromTable = handleExport(doMakeXLSXFromTable); -export const makeXLSXFromViewSection = handleExport(doMakeXLSXFromViewSection); +export const makeXLSXFromOptions = handleExport(doMakeXLSXFromOptions); function handleExport( - make: (a: ActiveDocSource, testDates: boolean, output: Stream, ...args: T) => Promise + make: (a: ActiveDocSource, testDates: boolean, output: Stream, ...args: T) => Promise ) { return async function({port, testDates, args}: {port: MessagePort, testDates: boolean, args: T}) { try { @@ -73,6 +73,23 @@ function bufferedPipe(stream: Stream, callback: (chunk: Buffer) => void, thresho stream.on('end', flush); } +export async function doMakeXLSXFromOptions( + activeDocSource: ActiveDocSource, + testDates: boolean, + stream: Stream, + options: ExportParameters +) { + const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options; + if (viewSectionId) { + return doMakeXLSXFromViewSection(activeDocSource, testDates, stream, viewSectionId, + sortOrder || null, filters || null, linkingFilter || null); + } else if (tableId) { + return doMakeXLSXFromTable(activeDocSource, testDates, stream, tableId); + } else { + return doMakeXLSX(activeDocSource, testDates, stream); + } +} + /** * Returns a XLSX stream of a view section that can be transformed or parsed. * @@ -86,14 +103,14 @@ async function doMakeXLSXFromViewSection( testDates: boolean, stream: Stream, viewSectionId: number, - sortOrder: number[], - filters: Filter[], - linkingFilter: FilterColValues, + sortOrder: number[] | null, + filters: Filter[] | null, + linkingFilter: FilterColValues | null, ) { const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters, linkingFilter); const {exportTable, end} = convertToExcel(stream, testDates); exportTable(data); - await end(); + return end(); } /** @@ -111,7 +128,7 @@ async function doMakeXLSXFromTable( const data = await doExportTable(activeDocSource, {tableId}); const {exportTable, end} = convertToExcel(stream, testDates); exportTable(data); - await end(); + return end(); } /** @@ -121,24 +138,33 @@ async function doMakeXLSX( activeDocSource: ActiveDocSource, testDates: boolean, stream: Stream, -): Promise { +): Promise { const {exportTable, end} = convertToExcel(stream, testDates); await doExportDoc(activeDocSource, async (table: ExportData) => exportTable(table)); - await end(); + return end(); } /** * Converts export data to an excel file. + * If a stream is provided, use it via the more memory-efficient + * WorkbookWriter, otherwise fall back on using a Workbook directly, + * and return a buffer. + * (The second option is for grist-static; at the time of writing + * WorkbookWriter doesn't appear to be available in a browser context). */ -function convertToExcel(stream: Stream, testDates: boolean): { +function convertToExcel(stream: Stream|undefined, testDates: boolean): { exportTable: (table: ExportData) => void, - end: () => Promise, + end: () => Promise, } { // Create workbook and add single sheet to it. Using the WorkbookWriter interface avoids // creating the entire Excel file in memory, which can be very memory-heavy. See // https://github.com/exceljs/exceljs#streaming-xlsx-writercontents. (The options useStyles and // useSharedStrings replicate more closely what was used previously.) - const wb = new ExcelWriteStream.xlsx.WorkbookWriter({useStyles: true, useSharedStrings: true, stream}); + // If there is no stream, write with a Workbook. + const wb: Workbook | ExcelWriteStream.xlsx.WorkbookWriter = stream ? + new ExcelWriteStream.xlsx.WorkbookWriter({ useStyles: true, useSharedStrings: true, stream }) : + new Workbook(); + const maybeCommit = stream ? (t: any) => t.commit() : (t: any) => {}; if (testDates) { // HACK: for testing, we will keep static dates const date = new Date(Date.UTC(2018, 11, 1, 0, 0, 0)); @@ -201,11 +227,16 @@ function convertToExcel(stream: Stream, testDates: boolean): { }); // Populate excel file with data for (const row of rowIds) { - ws.addRow(access.map((getter, c) => formatters[c].formatAny(getter(row)))).commit(); + maybeCommit(ws.addRow(access.map((getter, c) => formatters[c].formatAny(getter(row))))); + } + maybeCommit(ws); + } + async function end(): Promise { + if (!stream) { + return wb.xlsx.writeBuffer(); } - ws.commit(); + return maybeCommit(wb); } - function end() { return wb.commit(); } return {exportTable, end}; }