mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
add hooks for tweaking how downloads happen (for grist-static) (#665)
This makes three main changes: * Adds a hook to transform download links. * Adds a hook to add an externally created ActiveDoc to a DocManager. * Rejiggers XLSX export code so it can be used without streaming, which is currently tricky in a browser. Regular usage with node continues to use streaming. With these changes, I have a POC in hand that updates grist-static to support downloading CSVs, XLSXs, and .grist files.
This commit is contained in:
parent
f8c1bd612c
commit
585cf02f6c
@ -1,11 +1,22 @@
|
||||
import { UrlTweaks } from 'app/common/gristUrls';
|
||||
import { IAttrObj } from 'grainjs';
|
||||
|
||||
export interface IHooks {
|
||||
iframeAttributes?: Record<string, any>,
|
||||
fetch?: typeof fetch,
|
||||
baseURI?: string,
|
||||
urlTweaks?: UrlTweaks,
|
||||
|
||||
/**
|
||||
* Modify the attributes of an <a> dom element.
|
||||
* Convenient in grist-static to directly hook up a
|
||||
* download link with the function that provides the data.
|
||||
*/
|
||||
maybeModifyLinkAttrs(attrs: IAttrObj): IAttrObj;
|
||||
}
|
||||
|
||||
export const defaultHooks: IHooks = {
|
||||
maybeModifyLinkAttrs(attrs: IAttrObj) {
|
||||
return attrs;
|
||||
}
|
||||
};
|
||||
|
@ -3,6 +3,7 @@
|
||||
* the sample documents (those in the Support user's Examples & Templates workspace).
|
||||
*/
|
||||
|
||||
import {hooks} from 'app/client/Hooks';
|
||||
import {makeT} from 'app/client/lib/localization';
|
||||
import {AppModel, reportError} from 'app/client/models/AppModel';
|
||||
import {DocPageModel} from "app/client/models/DocPageModel";
|
||||
@ -302,14 +303,14 @@ export function downloadDocModal(doc: Document, pageModel: DocPageModel) {
|
||||
),
|
||||
cssModalButtons(
|
||||
dom.domComputed(use =>
|
||||
bigPrimaryButtonLink(`Download`, {
|
||||
bigPrimaryButtonLink(`Download`, hooks.maybeModifyLinkAttrs({
|
||||
href: pageModel.appModel.api.getDocAPI(doc.id).getDownloadUrl({
|
||||
template: use(selected) === "template",
|
||||
removeHistory: use(selected) === "nohistory" || use(selected) === "template",
|
||||
}),
|
||||
target: '_blank',
|
||||
download: ''
|
||||
},
|
||||
}),
|
||||
dom.on('click', () => {
|
||||
ctl.close();
|
||||
}),
|
||||
|
@ -1,3 +1,4 @@
|
||||
import {hooks} from 'app/client/Hooks';
|
||||
import {loadUserManager} from 'app/client/lib/imports';
|
||||
import {AppModel, reportError} from 'app/client/models/AppModel';
|
||||
import {DocInfo, DocPageModel} from 'app/client/models/DocPageModel';
|
||||
@ -255,12 +256,12 @@ function menuExports(doc: Document, pageModel: DocPageModel) {
|
||||
menuItem(() => downloadDocModal(doc, pageModel),
|
||||
menuIcon('Download'), t("Download..."), testId('tb-share-option'))
|
||||
),
|
||||
menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''},
|
||||
menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}),
|
||||
menuIcon('Download'), t("Export CSV"), testId('tb-share-option')),
|
||||
menuItemLink({
|
||||
menuItemLink(hooks.maybeModifyLinkAttrs({
|
||||
href: pageModel.appModel.api.getDocAPI(doc.id).getDownloadXlsxUrl(),
|
||||
target: '_blank', download: ''
|
||||
}, menuIcon('Download'), t("Export XLSX"), testId('tb-share-option')),
|
||||
}), menuIcon('Download'), t("Export XLSX"), testId('tb-share-option')),
|
||||
(!isFeatureEnabled("sendToDrive") ? null : menuItem(() => sendToDrive(doc, pageModel),
|
||||
menuIcon('Download'), t("Send to Google Drive"), testId('tb-share-option'))),
|
||||
];
|
||||
|
@ -1,3 +1,4 @@
|
||||
import {hooks} from 'app/client/Hooks';
|
||||
import {makeT} from 'app/client/lib/localization';
|
||||
import {allCommands} from 'app/client/components/commands';
|
||||
import {ViewSectionRec} from 'app/client/models/DocModel';
|
||||
@ -76,9 +77,9 @@ export function makeViewLayoutMenu(viewSection: ViewSectionRec, isReadonly: bool
|
||||
)
|
||||
),
|
||||
menuItemCmd(allCommands.printSection, t("Print widget"), testId('print-section')),
|
||||
menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''},
|
||||
menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}),
|
||||
t("Download as CSV"), testId('download-section')),
|
||||
menuItemLink({ href: gristDoc.getXlsxActiveViewLink(), target: '_blank', download: ''},
|
||||
menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getXlsxActiveViewLink(), target: '_blank', download: ''}),
|
||||
t("Download as XLSX"), testId('download-section')),
|
||||
dom.maybe((use) => ['detail', 'single'].includes(use(viewSection.parentKey)), () =>
|
||||
menuItemCmd(allCommands.editLayout, t("Edit Card Layout"),
|
||||
|
@ -49,7 +49,7 @@ import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
|
||||
import {DownloadOptions, parseExportParameters} from "app/server/lib/Export";
|
||||
import {downloadCSV} from "app/server/lib/ExportCSV";
|
||||
import {collectTableSchemaInFrictionlessFormat} from "app/server/lib/ExportTableSchema";
|
||||
import {downloadXLSX} from "app/server/lib/ExportXLSX";
|
||||
import {streamXLSX} from "app/server/lib/ExportXLSX";
|
||||
import {expressWrap} from 'app/server/lib/expressWrap';
|
||||
import {filterDocumentInPlace} from "app/server/lib/filterUtils";
|
||||
import {googleAuthTokenMiddleware} from "app/server/lib/GoogleAuth";
|
||||
@ -1911,3 +1911,14 @@ export interface WebhookSubscription {
|
||||
unsubscribeKey: string;
|
||||
webhookId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts `activeDoc` to XLSX and sends the converted data through `res`.
|
||||
*/
|
||||
export async function downloadXLSX(activeDoc: ActiveDoc, req: Request,
|
||||
res: Response, options: DownloadOptions) {
|
||||
const {filename} = options;
|
||||
res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
|
||||
res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx'));
|
||||
return streamXLSX(activeDoc, req, res, options);
|
||||
}
|
||||
|
@ -133,6 +133,14 @@ export class DocManager extends EventEmitter {
|
||||
return this.createNamedDoc(docSession, 'Untitled');
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an ActiveDoc created externally. This is a hook used by
|
||||
* grist-static.
|
||||
*/
|
||||
public addActiveDoc(docId: string, activeDoc: ActiveDoc) {
|
||||
this._activeDocs.set(docId, Promise.resolve(activeDoc));
|
||||
}
|
||||
|
||||
public async createNamedDoc(docSession: OptDocSession, docId: string): Promise<string> {
|
||||
const activeDoc: ActiveDoc = await this.createNewEmptyDoc(docSession, docId);
|
||||
await activeDoc.addInitialTable(docSession);
|
||||
|
@ -1,8 +1,7 @@
|
||||
/**
|
||||
* Overview of Excel exports, which now use worker-threads.
|
||||
*
|
||||
* 1. The flow starts with downloadXLSX() method called in the main thread (or streamXLSX() used for
|
||||
* Google Drive export).
|
||||
* 1. The flow starts with the streamXLSX() method called in the main thread.
|
||||
* 2. It uses the 'piscina' library to call a makeXLSX* method in a worker thread, registered in
|
||||
* workerExporter.ts, to export full doc, a table, or a section.
|
||||
* 3. Each of those methods calls a doMakeXLSX* method defined in that file. I.e. downloadXLSX()
|
||||
@ -12,11 +11,10 @@
|
||||
* 5. The resulting stream of Excel data is streamed back to the main thread using Rpc too.
|
||||
*/
|
||||
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
|
||||
import {ActiveDocSource, ActiveDocSourceDirect, DownloadOptions, ExportParameters} from 'app/server/lib/Export';
|
||||
import {ActiveDocSource, ActiveDocSourceDirect, ExportParameters} from 'app/server/lib/Export';
|
||||
import log from 'app/server/lib/log';
|
||||
import {addAbortHandler} from 'app/server/lib/requestUtils';
|
||||
import * as express from 'express';
|
||||
import contentDisposition from 'content-disposition';
|
||||
import {Rpc} from 'grain-rpc';
|
||||
import {AbortController} from 'node-abort-controller';
|
||||
import {Writable} from 'stream';
|
||||
@ -38,24 +36,12 @@ const exportPool = new Piscina({
|
||||
idleTimeout: 10_000, // Drop unused threads after 10s of inactivity.
|
||||
});
|
||||
|
||||
/**
|
||||
* Converts `activeDoc` to XLSX and sends the converted data through `res`.
|
||||
*/
|
||||
export async function downloadXLSX(activeDoc: ActiveDoc, req: express.Request,
|
||||
res: express.Response, options: DownloadOptions) {
|
||||
const {filename} = options;
|
||||
res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
|
||||
res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx'));
|
||||
return streamXLSX(activeDoc, req, res, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts `activeDoc` to XLSX and sends to the given outputStream.
|
||||
*/
|
||||
export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request,
|
||||
outputStream: Writable, options: ExportParameters) {
|
||||
log.debug(`Generating .xlsx file`);
|
||||
const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
|
||||
const testDates = (req.hostname === 'localhost');
|
||||
|
||||
const { port1, port2 } = new MessageChannel();
|
||||
@ -89,13 +75,7 @@ export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request,
|
||||
|
||||
// hanlding 3 cases : full XLSX export (full file), view xlsx export, table xlsx export
|
||||
try {
|
||||
if (viewSectionId) {
|
||||
await run('makeXLSXFromViewSection', viewSectionId, sortOrder, filters, linkingFilter);
|
||||
} else if (tableId) {
|
||||
await run('makeXLSXFromTable', tableId);
|
||||
} else {
|
||||
await run('makeXLSX');
|
||||
}
|
||||
await run('makeXLSXFromOptions', options);
|
||||
log.debug('XLSX file generated');
|
||||
} catch (e) {
|
||||
// We fiddle with errors in workerExporter to preserve extra properties like 'status'. Make
|
||||
|
@ -1,19 +1,19 @@
|
||||
import {PassThrough} from 'stream';
|
||||
import {FilterColValues} from "app/common/ActiveDocAPI";
|
||||
import {ActiveDocSource, doExportDoc, doExportSection, doExportTable, ExportData, Filter} from 'app/server/lib/Export';
|
||||
import {ActiveDocSource, doExportDoc, doExportSection, doExportTable,
|
||||
ExportData, ExportParameters, Filter} from 'app/server/lib/Export';
|
||||
import {createExcelFormatter} from 'app/server/lib/ExcelFormatter';
|
||||
import * as log from 'app/server/lib/log';
|
||||
import {Alignment, Border, stream as ExcelWriteStream, Fill} from 'exceljs';
|
||||
import {Alignment, Border, Buffer as ExcelBuffer, stream as ExcelWriteStream,
|
||||
Fill, Workbook} from 'exceljs';
|
||||
import {Rpc} from 'grain-rpc';
|
||||
import {Stream} from 'stream';
|
||||
import {MessagePort, threadId} from 'worker_threads';
|
||||
|
||||
export const makeXLSX = handleExport(doMakeXLSX);
|
||||
export const makeXLSXFromTable = handleExport(doMakeXLSXFromTable);
|
||||
export const makeXLSXFromViewSection = handleExport(doMakeXLSXFromViewSection);
|
||||
export const makeXLSXFromOptions = handleExport(doMakeXLSXFromOptions);
|
||||
|
||||
function handleExport<T extends any[]>(
|
||||
make: (a: ActiveDocSource, testDates: boolean, output: Stream, ...args: T) => Promise<void>
|
||||
make: (a: ActiveDocSource, testDates: boolean, output: Stream, ...args: T) => Promise<void|ExcelBuffer>
|
||||
) {
|
||||
return async function({port, testDates, args}: {port: MessagePort, testDates: boolean, args: T}) {
|
||||
try {
|
||||
@ -73,6 +73,23 @@ function bufferedPipe(stream: Stream, callback: (chunk: Buffer) => void, thresho
|
||||
stream.on('end', flush);
|
||||
}
|
||||
|
||||
export async function doMakeXLSXFromOptions(
|
||||
activeDocSource: ActiveDocSource,
|
||||
testDates: boolean,
|
||||
stream: Stream,
|
||||
options: ExportParameters
|
||||
) {
|
||||
const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
|
||||
if (viewSectionId) {
|
||||
return doMakeXLSXFromViewSection(activeDocSource, testDates, stream, viewSectionId,
|
||||
sortOrder || null, filters || null, linkingFilter || null);
|
||||
} else if (tableId) {
|
||||
return doMakeXLSXFromTable(activeDocSource, testDates, stream, tableId);
|
||||
} else {
|
||||
return doMakeXLSX(activeDocSource, testDates, stream);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a XLSX stream of a view section that can be transformed or parsed.
|
||||
*
|
||||
@ -86,14 +103,14 @@ async function doMakeXLSXFromViewSection(
|
||||
testDates: boolean,
|
||||
stream: Stream,
|
||||
viewSectionId: number,
|
||||
sortOrder: number[],
|
||||
filters: Filter[],
|
||||
linkingFilter: FilterColValues,
|
||||
sortOrder: number[] | null,
|
||||
filters: Filter[] | null,
|
||||
linkingFilter: FilterColValues | null,
|
||||
) {
|
||||
const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters, linkingFilter);
|
||||
const {exportTable, end} = convertToExcel(stream, testDates);
|
||||
exportTable(data);
|
||||
await end();
|
||||
return end();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -111,7 +128,7 @@ async function doMakeXLSXFromTable(
|
||||
const data = await doExportTable(activeDocSource, {tableId});
|
||||
const {exportTable, end} = convertToExcel(stream, testDates);
|
||||
exportTable(data);
|
||||
await end();
|
||||
return end();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -121,24 +138,33 @@ async function doMakeXLSX(
|
||||
activeDocSource: ActiveDocSource,
|
||||
testDates: boolean,
|
||||
stream: Stream,
|
||||
): Promise<void> {
|
||||
): Promise<void|ExcelBuffer> {
|
||||
const {exportTable, end} = convertToExcel(stream, testDates);
|
||||
await doExportDoc(activeDocSource, async (table: ExportData) => exportTable(table));
|
||||
await end();
|
||||
return end();
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts export data to an excel file.
|
||||
* If a stream is provided, use it via the more memory-efficient
|
||||
* WorkbookWriter, otherwise fall back on using a Workbook directly,
|
||||
* and return a buffer.
|
||||
* (The second option is for grist-static; at the time of writing
|
||||
* WorkbookWriter doesn't appear to be available in a browser context).
|
||||
*/
|
||||
function convertToExcel(stream: Stream, testDates: boolean): {
|
||||
function convertToExcel(stream: Stream|undefined, testDates: boolean): {
|
||||
exportTable: (table: ExportData) => void,
|
||||
end: () => Promise<void>,
|
||||
end: () => Promise<void|ExcelBuffer>,
|
||||
} {
|
||||
// Create workbook and add single sheet to it. Using the WorkbookWriter interface avoids
|
||||
// creating the entire Excel file in memory, which can be very memory-heavy. See
|
||||
// https://github.com/exceljs/exceljs#streaming-xlsx-writercontents. (The options useStyles and
|
||||
// useSharedStrings replicate more closely what was used previously.)
|
||||
const wb = new ExcelWriteStream.xlsx.WorkbookWriter({useStyles: true, useSharedStrings: true, stream});
|
||||
// If there is no stream, write with a Workbook.
|
||||
const wb: Workbook | ExcelWriteStream.xlsx.WorkbookWriter = stream ?
|
||||
new ExcelWriteStream.xlsx.WorkbookWriter({ useStyles: true, useSharedStrings: true, stream }) :
|
||||
new Workbook();
|
||||
const maybeCommit = stream ? (t: any) => t.commit() : (t: any) => {};
|
||||
if (testDates) {
|
||||
// HACK: for testing, we will keep static dates
|
||||
const date = new Date(Date.UTC(2018, 11, 1, 0, 0, 0));
|
||||
@ -201,11 +227,16 @@ function convertToExcel(stream: Stream, testDates: boolean): {
|
||||
});
|
||||
// Populate excel file with data
|
||||
for (const row of rowIds) {
|
||||
ws.addRow(access.map((getter, c) => formatters[c].formatAny(getter(row)))).commit();
|
||||
maybeCommit(ws.addRow(access.map((getter, c) => formatters[c].formatAny(getter(row)))));
|
||||
}
|
||||
ws.commit();
|
||||
maybeCommit(ws);
|
||||
}
|
||||
async function end(): Promise<void|ExcelBuffer> {
|
||||
if (!stream) {
|
||||
return wb.xlsx.writeBuffer();
|
||||
}
|
||||
return maybeCommit(wb);
|
||||
}
|
||||
function end() { return wb.commit(); }
|
||||
return {exportTable, end};
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user