add hooks for tweaking how downloads happen (for grist-static) (#665)

This makes three main changes:
  * Adds a hook to transform download links.
  * Adds a hook to add an externally created ActiveDoc to a DocManager.
  * Rejiggers XLSX export code so it can be used without streaming,
    which is currently tricky in a browser. Regular usage with node
    continues to use streaming.

With these changes, I have a POC in hand that updates grist-static
to support downloading CSVs, XLSXs, and .grist files.

app/client/DefaultHooks.ts

@@ -1,11 +1,22 @@
 import { UrlTweaks } from 'app/common/gristUrls';
+import { IAttrObj } from 'grainjs';
 
 export interface IHooks {
   iframeAttributes?: Record<string, any>,
   fetch?: typeof fetch,
   baseURI?: string,
   urlTweaks?: UrlTweaks,
+
+  /**
+   * Modify the attributes of an <a> dom element.
+   * Convenient in grist-static to directly hook up a
+   * download link with the function that provides the data.
+   */
+  maybeModifyLinkAttrs(attrs: IAttrObj): IAttrObj;
 }
 
 export const defaultHooks: IHooks = {
+  maybeModifyLinkAttrs(attrs: IAttrObj) {
+    return attrs;
+  }
 };

app/client/ui/MakeCopyMenu.ts

@@ -3,6 +3,7 @@
  * the sample documents (those in the Support user's Examples & Templates workspace).
  */
 
+import {hooks} from 'app/client/Hooks';
 import {makeT} from 'app/client/lib/localization';
 import {AppModel, reportError} from 'app/client/models/AppModel';
 import {DocPageModel} from "app/client/models/DocPageModel";
@@ -302,14 +303,14 @@ export function downloadDocModal(doc: Document, pageModel: DocPageModel) {
       ),
       cssModalButtons(
         dom.domComputed(use =>
-          bigPrimaryButtonLink(`Download`, {
+          bigPrimaryButtonLink(`Download`, hooks.maybeModifyLinkAttrs({
               href: pageModel.appModel.api.getDocAPI(doc.id).getDownloadUrl({
                 template: use(selected) === "template",
                 removeHistory: use(selected) === "nohistory" || use(selected) === "template",
               }),
               target: '_blank',
               download: ''
-            },
+            }),
             dom.on('click', () => {
               ctl.close();
             }),

app/client/ui/ShareMenu.ts

@@ -1,3 +1,4 @@
+import {hooks} from 'app/client/Hooks';
 import {loadUserManager} from 'app/client/lib/imports';
 import {AppModel, reportError} from 'app/client/models/AppModel';
 import {DocInfo, DocPageModel} from 'app/client/models/DocPageModel';
@@ -255,12 +256,12 @@ function menuExports(doc: Document, pageModel: DocPageModel) {
         menuItem(() => downloadDocModal(doc, pageModel),
         menuIcon('Download'), t("Download..."), testId('tb-share-option'))
     ),
-    menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''},
+    menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}),
       menuIcon('Download'), t("Export CSV"), testId('tb-share-option')),
-    menuItemLink({
+    menuItemLink(hooks.maybeModifyLinkAttrs({
       href: pageModel.appModel.api.getDocAPI(doc.id).getDownloadXlsxUrl(),
       target: '_blank', download: ''
-    }, menuIcon('Download'), t("Export XLSX"), testId('tb-share-option')),
+    }), menuIcon('Download'), t("Export XLSX"), testId('tb-share-option')),
     (!isFeatureEnabled("sendToDrive") ? null : menuItem(() => sendToDrive(doc, pageModel),
       menuIcon('Download'), t("Send to Google Drive"), testId('tb-share-option'))),
   ];

app/client/ui/ViewLayoutMenu.ts

@@ -1,3 +1,4 @@
+import {hooks} from 'app/client/Hooks';
 import {makeT} from 'app/client/lib/localization';
 import {allCommands} from 'app/client/components/commands';
 import {ViewSectionRec} from 'app/client/models/DocModel';
@@ -76,9 +77,9 @@ export function makeViewLayoutMenu(viewSection: ViewSectionRec, isReadonly: bool
       )
     ),
     menuItemCmd(allCommands.printSection, t("Print widget"), testId('print-section')),
-    menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''},
+    menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}),
       t("Download as CSV"), testId('download-section')),
-    menuItemLink({ href: gristDoc.getXlsxActiveViewLink(), target: '_blank', download: ''},
+    menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getXlsxActiveViewLink(), target: '_blank', download: ''}),
       t("Download as XLSX"), testId('download-section')),
     dom.maybe((use) => ['detail', 'single'].includes(use(viewSection.parentKey)), () =>
       menuItemCmd(allCommands.editLayout, t("Edit Card Layout"),

app/server/lib/DocApi.ts

@@ -49,7 +49,7 @@ import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
 import {DownloadOptions, parseExportParameters} from "app/server/lib/Export";
 import {downloadCSV} from "app/server/lib/ExportCSV";
 import {collectTableSchemaInFrictionlessFormat} from "app/server/lib/ExportTableSchema";
-import {downloadXLSX} from "app/server/lib/ExportXLSX";
+import {streamXLSX} from "app/server/lib/ExportXLSX";
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {filterDocumentInPlace} from "app/server/lib/filterUtils";
 import {googleAuthTokenMiddleware} from "app/server/lib/GoogleAuth";
@@ -1911,3 +1911,14 @@ export interface WebhookSubscription {
   unsubscribeKey: string;
   webhookId: string;
 }
+
+/**
+ * Converts `activeDoc` to XLSX and sends the converted data through `res`.
+ */
+export async function downloadXLSX(activeDoc: ActiveDoc, req: Request,
+                                   res: Response, options: DownloadOptions) {
+  const {filename} = options;
+  res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
+  res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx'));
+  return streamXLSX(activeDoc, req, res, options);
+}

app/server/lib/DocManager.ts

@@ -133,6 +133,14 @@ export class DocManager extends EventEmitter {
     return this.createNamedDoc(docSession, 'Untitled');
   }
 
+  /**
+   * Add an ActiveDoc created externally. This is a hook used by
+   * grist-static.
+   */
+  public addActiveDoc(docId: string, activeDoc: ActiveDoc) {
+    this._activeDocs.set(docId, Promise.resolve(activeDoc));
+  }
+
   public async createNamedDoc(docSession: OptDocSession, docId: string): Promise<string> {
     const activeDoc: ActiveDoc = await this.createNewEmptyDoc(docSession, docId);
     await activeDoc.addInitialTable(docSession);

app/server/lib/ExportXLSX.ts

@@ -1,8 +1,7 @@
 /**
  * Overview of Excel exports, which now use worker-threads.
  *
- * 1. The flow starts with downloadXLSX() method called in the main thread (or streamXLSX() used for
+ * 1. The flow starts with the streamXLSX() method called in the main thread.
- *    Google Drive export).
  * 2. It uses the 'piscina' library to call a makeXLSX* method in a worker thread, registered in
  *    workerExporter.ts, to export full doc, a table, or a section.
  * 3. Each of those methods calls a doMakeXLSX* method defined in that file. I.e. downloadXLSX()
@@ -12,11 +11,10 @@
  * 5. The resulting stream of Excel data is streamed back to the main thread using Rpc too.
  */
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {ActiveDocSource, ActiveDocSourceDirect, DownloadOptions, ExportParameters} from 'app/server/lib/Export';
+import {ActiveDocSource, ActiveDocSourceDirect, ExportParameters} from 'app/server/lib/Export';
 import log from 'app/server/lib/log';
 import {addAbortHandler} from 'app/server/lib/requestUtils';
 import * as express from 'express';
-import contentDisposition from 'content-disposition';
 import {Rpc} from 'grain-rpc';
 import {AbortController} from 'node-abort-controller';
 import {Writable} from 'stream';
@@ -38,24 +36,12 @@ const exportPool = new Piscina({
   idleTimeout: 10_000,    // Drop unused threads after 10s of inactivity.
 });
 
-/**
- * Converts `activeDoc` to XLSX and sends the converted data through `res`.
- */
-export async function downloadXLSX(activeDoc: ActiveDoc, req: express.Request,
-                                   res: express.Response, options: DownloadOptions) {
-  const {filename} = options;
-  res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
-  res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx'));
-  return streamXLSX(activeDoc, req, res, options);
-}
-
 /**
  * Converts `activeDoc` to XLSX and sends to the given outputStream.
  */
 export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request,
                                  outputStream: Writable, options: ExportParameters) {
   log.debug(`Generating .xlsx file`);
-  const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
   const testDates = (req.hostname === 'localhost');
 
   const { port1, port2 } = new MessageChannel();
@@ -89,13 +75,7 @@ export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request,
 
     // hanlding 3 cases : full XLSX export (full file), view xlsx export, table xlsx export
     try {
-      if (viewSectionId) {
+      await run('makeXLSXFromOptions', options);
-        await run('makeXLSXFromViewSection', viewSectionId, sortOrder, filters, linkingFilter);
-      } else if (tableId) {
-        await run('makeXLSXFromTable', tableId);
-      } else {
-        await run('makeXLSX');
-      }
       log.debug('XLSX file generated');
     } catch (e) {
       // We fiddle with errors in workerExporter to preserve extra properties like 'status'. Make

app/server/lib/workerExporter.ts

@@ -1,19 +1,19 @@
 import {PassThrough} from 'stream';
 import {FilterColValues} from "app/common/ActiveDocAPI";
-import {ActiveDocSource, doExportDoc, doExportSection, doExportTable, ExportData, Filter} from 'app/server/lib/Export';
+import {ActiveDocSource, doExportDoc, doExportSection, doExportTable,
+        ExportData, ExportParameters, Filter} from 'app/server/lib/Export';
 import {createExcelFormatter} from 'app/server/lib/ExcelFormatter';
 import * as log from 'app/server/lib/log';
-import {Alignment, Border, stream as ExcelWriteStream, Fill} from 'exceljs';
+import {Alignment, Border, Buffer as ExcelBuffer, stream as ExcelWriteStream,
+        Fill, Workbook} from 'exceljs';
 import {Rpc} from 'grain-rpc';
 import {Stream} from 'stream';
 import {MessagePort, threadId} from 'worker_threads';
 
-export const makeXLSX = handleExport(doMakeXLSX);
+export const makeXLSXFromOptions = handleExport(doMakeXLSXFromOptions);
-export const makeXLSXFromTable = handleExport(doMakeXLSXFromTable);
-export const makeXLSXFromViewSection = handleExport(doMakeXLSXFromViewSection);
 
 function handleExport<T extends any[]>(
-  make: (a: ActiveDocSource, testDates: boolean, output: Stream, ...args: T) => Promise<void>
+  make: (a: ActiveDocSource, testDates: boolean, output: Stream, ...args: T) => Promise<void|ExcelBuffer>
 ) {
   return async function({port, testDates, args}: {port: MessagePort, testDates: boolean, args: T}) {
     try {
@@ -73,6 +73,23 @@ function bufferedPipe(stream: Stream, callback: (chunk: Buffer) => void, thresho
   stream.on('end', flush);
 }
 
+export async function doMakeXLSXFromOptions(
+  activeDocSource: ActiveDocSource,
+  testDates: boolean,
+  stream: Stream,
+  options: ExportParameters
+) {
+  const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
+  if (viewSectionId) {
+    return doMakeXLSXFromViewSection(activeDocSource, testDates, stream, viewSectionId,
+      sortOrder || null, filters || null, linkingFilter || null);
+  } else if (tableId) {
+    return doMakeXLSXFromTable(activeDocSource, testDates, stream, tableId);
+  } else {
+    return doMakeXLSX(activeDocSource, testDates, stream);
+  }
+}
+
 /**
  * Returns a XLSX stream of a view section that can be transformed or parsed.
  *
@@ -86,14 +103,14 @@ async function doMakeXLSXFromViewSection(
   testDates: boolean,
   stream: Stream,
   viewSectionId: number,
-  sortOrder: number[],
+  sortOrder: number[] | null,
-  filters: Filter[],
+  filters: Filter[] | null,
-  linkingFilter: FilterColValues,
+  linkingFilter: FilterColValues | null,
 ) {
   const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters, linkingFilter);
   const {exportTable, end} = convertToExcel(stream, testDates);
   exportTable(data);
-  await end();
+  return end();
 }
 
 /**
@@ -111,7 +128,7 @@ async function doMakeXLSXFromTable(
   const data = await doExportTable(activeDocSource, {tableId});
   const {exportTable, end} = convertToExcel(stream, testDates);
   exportTable(data);
-  await end();
+  return end();
 }
 
 /**
@@ -121,24 +138,33 @@ async function doMakeXLSX(
   activeDocSource: ActiveDocSource,
   testDates: boolean,
   stream: Stream,
-): Promise<void> {
+): Promise<void|ExcelBuffer> {
   const {exportTable, end} = convertToExcel(stream, testDates);
   await doExportDoc(activeDocSource, async (table: ExportData) => exportTable(table));
-  await end();
+  return end();
 }
 
 /**
  * Converts export data to an excel file.
+ * If a stream is provided, use it via the more memory-efficient
+ * WorkbookWriter, otherwise fall back on using a Workbook directly,
+ * and return a buffer.
+ * (The second option is for grist-static; at the time of writing
+ * WorkbookWriter doesn't appear to be available in a browser context).
  */
-function convertToExcel(stream: Stream, testDates: boolean): {
+function convertToExcel(stream: Stream|undefined, testDates: boolean): {
   exportTable: (table: ExportData) => void,
-  end: () => Promise<void>,
+  end: () => Promise<void|ExcelBuffer>,
 } {
   // Create workbook and add single sheet to it. Using the WorkbookWriter interface avoids
   // creating the entire Excel file in memory, which can be very memory-heavy. See
   // https://github.com/exceljs/exceljs#streaming-xlsx-writercontents. (The options useStyles and
   // useSharedStrings replicate more closely what was used previously.)
-  const wb = new ExcelWriteStream.xlsx.WorkbookWriter({useStyles: true, useSharedStrings: true, stream});
+  // If there is no stream, write with a Workbook.
+  const wb: Workbook | ExcelWriteStream.xlsx.WorkbookWriter = stream ?
+      new ExcelWriteStream.xlsx.WorkbookWriter({ useStyles: true, useSharedStrings: true, stream }) :
+      new Workbook();
+  const maybeCommit = stream ? (t: any) => t.commit() : (t: any) => {};
   if (testDates) {
     // HACK: for testing, we will keep static dates
     const date = new Date(Date.UTC(2018, 11, 1, 0, 0, 0));
@@ -201,11 +227,16 @@ function convertToExcel(stream: Stream, testDates: boolean): {
     });
     // Populate excel file with data
     for (const row of rowIds) {
-      ws.addRow(access.map((getter, c) => formatters[c].formatAny(getter(row)))).commit();
+      maybeCommit(ws.addRow(access.map((getter, c) => formatters[c].formatAny(getter(row)))));
     }
-    ws.commit();
+    maybeCommit(ws);
+  }
+  async function end(): Promise<void|ExcelBuffer> {
+    if (!stream) {
+      return wb.xlsx.writeBuffer();
+    }
+    return maybeCommit(wb);
   }
-  function end() { return wb.commit(); }
   return {exportTable, end};
 }