(core) updates from grist-core

app/server/lib/DocApi.ts

@@ -50,7 +50,7 @@ import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
 import {DownloadOptions, parseExportParameters} from "app/server/lib/Export";
 import {downloadCSV} from "app/server/lib/ExportCSV";
 import {collectTableSchemaInFrictionlessFormat} from "app/server/lib/ExportTableSchema";
-import {downloadXLSX} from "app/server/lib/ExportXLSX";
+import {streamXLSX} from "app/server/lib/ExportXLSX";
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {filterDocumentInPlace} from "app/server/lib/filterUtils";
 import {googleAuthTokenMiddleware} from "app/server/lib/GoogleAuth";
@@ -173,6 +173,7 @@ export class DocWorkerApi {
     const canView = expressWrap(this._assertAccess.bind(this, 'viewers', false));
     // check document exists (not soft deleted) and user can edit it
     const canEdit = expressWrap(this._assertAccess.bind(this, 'editors', false));
+    const checkAnonymousCreation = expressWrap(this._checkAnonymousCreation.bind(this));
     const isOwner = expressWrap(this._assertAccess.bind(this, 'owners', false));
     // check user can edit document, with soft-deleted documents being acceptable
     const canEditMaybeRemoved = expressWrap(this._assertAccess.bind(this, 'editors', true));
@@ -1241,7 +1242,7 @@ export class DocWorkerApi {
      *
      * TODO: unify this with the other document creation and import endpoints.
      */
-    this._app.post('/api/docs', expressWrap(async (req, res) => {
+    this._app.post('/api/docs', checkAnonymousCreation, expressWrap(async (req, res) => {
       const userId = getUserId(req);
 
       let uploadId: number|undefined;
@@ -1522,6 +1523,17 @@ export class DocWorkerApi {
     return await this._dbManager.increaseUsage(getDocScope(req), limit, {delta: 1});
   }
 
+  /**
+   * Disallow document creation for anonymous users if GRIST_ANONYMOUS_CREATION is set to false.
+   */
+  private async _checkAnonymousCreation(req: Request, res: Response, next: NextFunction) {
+    const isAnonPlayground = isAffirmative(process.env.GRIST_ANON_PLAYGROUND ?? true);
+    if (isAnonymousUser(req) && !isAnonPlayground) {
+      throw new ApiError('Anonymous document creation is disabled', 403);
+    }
+    next();
+  }
+
   private async _assertAccess(role: 'viewers'|'editors'|'owners'|null, allowRemoved: boolean,
                               req: Request, res: Response, next: NextFunction) {
     const scope = getDocScope(req);
@@ -1969,3 +1981,14 @@ export interface WebhookSubscription {
   unsubscribeKey: string;
   webhookId: string;
 }
+
+/**
+ * Converts `activeDoc` to XLSX and sends the converted data through `res`.
+ */
+export async function downloadXLSX(activeDoc: ActiveDoc, req: Request,
+                                   res: Response, options: DownloadOptions) {
+  const {filename} = options;
+  res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
+  res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx'));
+  return streamXLSX(activeDoc, req, res, options);
+}

app/server/lib/DocManager.ts

@@ -133,6 +133,14 @@ export class DocManager extends EventEmitter {
     return this.createNamedDoc(docSession, 'Untitled');
   }
 
+  /**
+   * Add an ActiveDoc created externally. This is a hook used by
+   * grist-static.
+   */
+  public addActiveDoc(docId: string, activeDoc: ActiveDoc) {
+    this._activeDocs.set(docId, Promise.resolve(activeDoc));
+  }
+
   public async createNamedDoc(docSession: OptDocSession, docId: string): Promise<string> {
     const activeDoc: ActiveDoc = await this.createNewEmptyDoc(docSession, docId);
     await activeDoc.addInitialTable(docSession);

app/server/lib/ExportXLSX.ts

@@ -1,8 +1,7 @@
 /**
  * Overview of Excel exports, which now use worker-threads.
  *
- * 1. The flow starts with downloadXLSX() method called in the main thread (or streamXLSX() used for
- *    Google Drive export).
+ * 1. The flow starts with the streamXLSX() method called in the main thread.
  * 2. It uses the 'piscina' library to call a makeXLSX* method in a worker thread, registered in
  *    workerExporter.ts, to export full doc, a table, or a section.
  * 3. Each of those methods calls a doMakeXLSX* method defined in that file. I.e. downloadXLSX()
@@ -12,11 +11,10 @@
  * 5. The resulting stream of Excel data is streamed back to the main thread using Rpc too.
  */
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {ActiveDocSource, ActiveDocSourceDirect, DownloadOptions, ExportParameters} from 'app/server/lib/Export';
+import {ActiveDocSource, ActiveDocSourceDirect, ExportParameters} from 'app/server/lib/Export';
 import log from 'app/server/lib/log';
 import {addAbortHandler} from 'app/server/lib/requestUtils';
 import * as express from 'express';
-import contentDisposition from 'content-disposition';
 import {Rpc} from 'grain-rpc';
 import {AbortController} from 'node-abort-controller';
 import {Writable} from 'stream';
@@ -38,24 +36,12 @@ const exportPool = new Piscina({
   idleTimeout: 10_000,    // Drop unused threads after 10s of inactivity.
 });
 
-/**
- * Converts `activeDoc` to XLSX and sends the converted data through `res`.
- */
-export async function downloadXLSX(activeDoc: ActiveDoc, req: express.Request,
-                                   res: express.Response, options: DownloadOptions) {
-  const {filename} = options;
-  res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
-  res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx'));
-  return streamXLSX(activeDoc, req, res, options);
-}
-
 /**
  * Converts `activeDoc` to XLSX and sends to the given outputStream.
  */
 export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request,
                                  outputStream: Writable, options: ExportParameters) {
   log.debug(`Generating .xlsx file`);
-  const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
   const testDates = (req.hostname === 'localhost');
 
   const { port1, port2 } = new MessageChannel();
@@ -89,13 +75,7 @@ export async function streamXLSX(activeDoc: ActiveDoc, req: express.Request,
 
     // hanlding 3 cases : full XLSX export (full file), view xlsx export, table xlsx export
     try {
-      if (viewSectionId) {
-        await run('makeXLSXFromViewSection', viewSectionId, sortOrder, filters, linkingFilter);
-      } else if (tableId) {
-        await run('makeXLSXFromTable', tableId);
-      } else {
-        await run('makeXLSX');
-      }
+      await run('makeXLSXFromOptions', options);
       log.debug('XLSX file generated');
     } catch (e) {
       // We fiddle with errors in workerExporter to preserve extra properties like 'status'. Make

app/server/lib/FlexServer.ts

@@ -852,10 +852,11 @@ export class FlexServer implements GristServer {
       baseDomain: this._defaultBaseDomain,
     });
 
-    const isForced = appSettings.section('login').flag('forced').readBool({
+    const forceLogin = appSettings.section('login').flag('forced').readBool({
       envVar: 'GRIST_FORCE_LOGIN',
     });
-    const forcedLoginMiddleware = isForced ? this._redirectToLoginWithoutExceptionsMiddleware : noop;
+
+    const forcedLoginMiddleware = forceLogin ? this._redirectToLoginWithoutExceptionsMiddleware : noop;
 
     const welcomeNewUser: express.RequestHandler = isSingleUserMode() ?
       (req, res, next) => next() :

app/server/lib/sendAppPage.ts

@@ -56,6 +56,7 @@ export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig
     helpCenterUrl: process.env.GRIST_HELP_CENTER || "https://support.getgrist.com",
     pathOnly,
     supportAnon: shouldSupportAnon(),
+    enableAnonPlayground: isAffirmative(process.env.GRIST_ANON_PLAYGROUND ?? true),
     supportEngines: getSupportedEngineChoices(),
     features: getFeatures(),
     pageTitleSuffix: configuredPageTitleSuffix(),

app/server/lib/workerExporter.ts

@@ -1,19 +1,19 @@
 import {PassThrough} from 'stream';
 import {FilterColValues} from "app/common/ActiveDocAPI";
-import {ActiveDocSource, doExportDoc, doExportSection, doExportTable, ExportData, Filter} from 'app/server/lib/Export';
+import {ActiveDocSource, doExportDoc, doExportSection, doExportTable,
+        ExportData, ExportParameters, Filter} from 'app/server/lib/Export';
 import {createExcelFormatter} from 'app/server/lib/ExcelFormatter';
 import * as log from 'app/server/lib/log';
-import {Alignment, Border, stream as ExcelWriteStream, Fill} from 'exceljs';
+import {Alignment, Border, Buffer as ExcelBuffer, stream as ExcelWriteStream,
+        Fill, Workbook} from 'exceljs';
 import {Rpc} from 'grain-rpc';
 import {Stream} from 'stream';
 import {MessagePort, threadId} from 'worker_threads';
 
-export const makeXLSX = handleExport(doMakeXLSX);
-export const makeXLSXFromTable = handleExport(doMakeXLSXFromTable);
-export const makeXLSXFromViewSection = handleExport(doMakeXLSXFromViewSection);
+export const makeXLSXFromOptions = handleExport(doMakeXLSXFromOptions);
 
 function handleExport<T extends any[]>(
-  make: (a: ActiveDocSource, testDates: boolean, output: Stream, ...args: T) => Promise<void>
+  make: (a: ActiveDocSource, testDates: boolean, output: Stream, ...args: T) => Promise<void|ExcelBuffer>
 ) {
   return async function({port, testDates, args}: {port: MessagePort, testDates: boolean, args: T}) {
     try {
@@ -73,6 +73,23 @@ function bufferedPipe(stream: Stream, callback: (chunk: Buffer) => void, thresho
   stream.on('end', flush);
 }
 
+export async function doMakeXLSXFromOptions(
+  activeDocSource: ActiveDocSource,
+  testDates: boolean,
+  stream: Stream,
+  options: ExportParameters
+) {
+  const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
+  if (viewSectionId) {
+    return doMakeXLSXFromViewSection(activeDocSource, testDates, stream, viewSectionId,
+      sortOrder || null, filters || null, linkingFilter || null);
+  } else if (tableId) {
+    return doMakeXLSXFromTable(activeDocSource, testDates, stream, tableId);
+  } else {
+    return doMakeXLSX(activeDocSource, testDates, stream);
+  }
+}
+
 /**
  * Returns a XLSX stream of a view section that can be transformed or parsed.
  *
@@ -86,14 +103,14 @@ async function doMakeXLSXFromViewSection(
   testDates: boolean,
   stream: Stream,
   viewSectionId: number,
-  sortOrder: number[],
-  filters: Filter[],
-  linkingFilter: FilterColValues,
+  sortOrder: number[] | null,
+  filters: Filter[] | null,
+  linkingFilter: FilterColValues | null,
 ) {
   const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters, linkingFilter);
   const {exportTable, end} = convertToExcel(stream, testDates);
   exportTable(data);
-  await end();
+  return end();
 }
 
 /**
@@ -111,7 +128,7 @@ async function doMakeXLSXFromTable(
   const data = await doExportTable(activeDocSource, {tableId});
   const {exportTable, end} = convertToExcel(stream, testDates);
   exportTable(data);
-  await end();
+  return end();
 }
 
 /**
@@ -121,24 +138,33 @@ async function doMakeXLSX(
   activeDocSource: ActiveDocSource,
   testDates: boolean,
   stream: Stream,
-): Promise<void> {
+): Promise<void|ExcelBuffer> {
   const {exportTable, end} = convertToExcel(stream, testDates);
   await doExportDoc(activeDocSource, async (table: ExportData) => exportTable(table));
-  await end();
+  return end();
 }
 
 /**
  * Converts export data to an excel file.
+ * If a stream is provided, use it via the more memory-efficient
+ * WorkbookWriter, otherwise fall back on using a Workbook directly,
+ * and return a buffer.
+ * (The second option is for grist-static; at the time of writing
+ * WorkbookWriter doesn't appear to be available in a browser context).
  */
-function convertToExcel(stream: Stream, testDates: boolean): {
+function convertToExcel(stream: Stream|undefined, testDates: boolean): {
   exportTable: (table: ExportData) => void,
-  end: () => Promise<void>,
+  end: () => Promise<void|ExcelBuffer>,
 } {
   // Create workbook and add single sheet to it. Using the WorkbookWriter interface avoids
   // creating the entire Excel file in memory, which can be very memory-heavy. See
   // https://github.com/exceljs/exceljs#streaming-xlsx-writercontents. (The options useStyles and
   // useSharedStrings replicate more closely what was used previously.)
-  const wb = new ExcelWriteStream.xlsx.WorkbookWriter({useStyles: true, useSharedStrings: true, stream});
+  // If there is no stream, write with a Workbook.
+  const wb: Workbook | ExcelWriteStream.xlsx.WorkbookWriter = stream ?
+      new ExcelWriteStream.xlsx.WorkbookWriter({ useStyles: true, useSharedStrings: true, stream }) :
+      new Workbook();
+  const maybeCommit = stream ? (t: any) => t.commit() : (t: any) => {};
   if (testDates) {
     // HACK: for testing, we will keep static dates
     const date = new Date(Date.UTC(2018, 11, 1, 0, 0, 0));
@@ -201,11 +227,16 @@ function convertToExcel(stream: Stream, testDates: boolean): {
     });
     // Populate excel file with data
     for (const row of rowIds) {
-      ws.addRow(access.map((getter, c) => formatters[c].formatAny(getter(row)))).commit();
+      maybeCommit(ws.addRow(access.map((getter, c) => formatters[c].formatAny(getter(row)))));
     }
-    ws.commit();
+    maybeCommit(ws);
+  }
+  async function end(): Promise<void|ExcelBuffer> {
+    if (!stream) {
+      return wb.xlsx.writeBuffer();
+    }
+    return maybeCommit(wb);
   }
-  function end() { return wb.commit(); }
   return {exportTable, end};
 }