(core) Export to Excel and Send to drive

Summary:
Implementing export to excel and send to Google Drive feature.

As part of this feature few things were implemented:
- Server side google authentication exposed on url: (docs, docs-s, or localhost:8080)/auth/google
- Exporting grist documents as an excel file (xlsx)
- Storing exported grist document (in excel format) in Google Drive as a spreadsheet document.

Server side google authentication requires one new environmental variables
- GOOGLE_CLIENT_SECRET (required) used by authentication handler

Test Plan: Browser tests for exporting to excel.

Reviewers: paulfitz, dsagal

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2924
pull/9/head
Jarosław Sadziński 3 years ago
parent 9cc034f606
commit 08295a696b

@ -601,6 +601,14 @@ export class GristDoc extends DisposableWithEvents {
);
}
public getXlsxLink() {
const params = {
...this.docComm.getUrlParams(),
title: this.docPageModel.currentDocTitle.get(),
};
return this.docComm.docUrl(`gen_xlsx`) + '?' + encodeQueryParams(params);
}
public getCsvLink() {
const filters = this.viewModel.activeSection.peek().filteredFields.get().map(field=> ({
colRef : field.colRef.peek(),
@ -614,7 +622,7 @@ export class GristDoc extends DisposableWithEvents {
activeSortSpec: JSON.stringify(this.viewModel.activeSection().activeSortSpec()),
filters : JSON.stringify(filters),
};
return this.docComm.docUrl('gen_csv') + '?' + encodeQueryParams(params);
return this.docComm.docUrl(`gen_csv`) + '?' + encodeQueryParams(params);
}
public hasGranularAccessRules(): boolean {

@ -3,6 +3,7 @@ import {AppModel, reportError} from 'app/client/models/AppModel';
import {DocInfo, DocPageModel} from 'app/client/models/DocPageModel';
import {docUrl, urlState} from 'app/client/models/gristUrlState';
import {makeCopy, replaceTrunkWithFork} from 'app/client/ui/MakeCopyMenu';
import {sendToDrive} from 'app/client/ui/sendToDrive';
import {cssHoverCircle, cssTopBarBtn} from 'app/client/ui/TopBarCss';
import {primaryButton} from 'app/client/ui2018/buttons';
import {colors, mediaXSmall, testId} from 'app/client/ui2018/cssVars';
@ -222,6 +223,10 @@ function menuExports(doc: Document, pageModel: DocPageModel) {
),
menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''},
menuIcon('Download'), 'Export CSV', testId('tb-share-option')),
menuItemLink({ href: gristDoc.getXlsxLink(), target: '_blank', download: ''},
menuIcon('Download'), 'Export XLSX', testId('tb-share-option')),
menuItem(() => sendToDrive(doc, pageModel),
menuIcon('Download'), 'Send to Google Drive', testId('tb-share-option')),
];
}

@ -0,0 +1,106 @@
import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
import {getHomeUrl} from 'app/client/models/AppModel';
import {reportError} from 'app/client/models/errors';
import {spinnerModal} from 'app/client/ui2018/modals';
import type { DocPageModel } from 'app/client/models/DocPageModel';
import type { Document } from 'app/common/UserAPI';
import type { Disposable } from 'grainjs';
const G = getBrowserGlobals('window');
/**
* Generates Google Auth endpoint (exposed by Grist) url. For example:
* https://docs.getgrist.com/auth/google
* @param scope Requested access scope for Google Services
* https://developers.google.com/identity/protocols/oauth2/scopes
*/
function getGoogleAuthEndpoint(scope?: string) {
return new URL(`auth/google?scope=${scope || ''}`, getHomeUrl()).href;
}
/**
* Sends xlsx file to Google Drive. It first authenticates with Google to get encrypted access token,
* then it calls "send-to-drive" api endpoint to upload xlsx file to drive and finally it redirects
* to the created spreadsheet.
* Code that is received from Google contains encrypted access token, server is able to decrypt it
* using GOOGLE_CLIENT_SECRET key.
*/
export function sendToDrive(doc: Document, pageModel: DocPageModel) {
// Get current document - it will be used to remove popup listener.
const gristDoc = pageModel.gristDoc.get();
// Sanity check - gristDoc should be always present
if (!gristDoc) { throw new Error("Grist document is not present in Page Model"); }
// Create send to google drive handler (it will return a spreadsheet url).
const send = (code: string) =>
// Decorate it with a spinner
spinnerModal('Sending file to Google Drive',
pageModel.appModel.api.getDocAPI(doc.id)
.sendToDrive(code, pageModel.currentDocTitle.get())
);
// Compute google auth server endpoint (grist endpoint for server side google authentication).
// This endpoint will redirect user to Google Consent screen and after Google sends a response,
// it will render a page (/static/message.html) that will post a message containing message
// from Google. Message will be an object { code, error }. We will use the code to invoke
// "send-to-drive" api endpoint - that will actually send the xlsx file to Google Drive.
const authLink = getGoogleAuthEndpoint();
const authWindow = openPopup(authLink);
attachListener(gristDoc, authWindow, async (event: MessageEvent) => {
// For the first message (we expect only a single message) close the window.
authWindow.close();
// Check response from the popup
const response = (event.data || {}) as { code?: string, error?: string };
// - when user declined, do nothing,
if (response.error === "access_denied") { return; }
// - when there is no response code or error code is different from what we expected - report to user.
if (!response.code) { reportError(response.error || "Unrecognized or empty error code"); return; }
// Send file to Google Drive.
try {
const { url } = await send(response.code);
G.window.location.assign(url);
} catch (err) {
reportError(err);
}
});
}
// Helper function that attaches a handler to message event from a popup window.
function attachListener(owner: Disposable, popup: Window, listener: (e: MessageEvent) => any) {
const wrapped = (e: MessageEvent) => {
// Listen to events only from our window.
if (e.source !== popup) { return; }
// In case when Grist was reloaded or user navigated away - do nothing.
if (owner.isDisposed()) { return; }
listener(e);
};
G.window.addEventListener('message', wrapped);
owner.onDispose(() => {
G.window.removeEventListener('message', wrapped);
});
}
function openPopup(url: string): Window {
// Center window on desktop
// https://stackoverflow.com/questions/16363474/window-open-on-a-multi-monitor-dual-monitor-system-where-does-window-pop-up
const width = 600;
const height = 650;
const left = window.screenX + (screen.width - width) / 2;
const top = (screen.height - height) / 4;
let windowFeatures = `top=${top},left=${left},menubar=no,location=no,` +
`resizable=yes,scrollbars=yes,status=yes,height=${height},width=${width}`;
// If window will be too large (for example on mobile) - open as a new tab
if (screen.width <= width || screen.height <= height) {
windowFeatures = '';
}
const authWindow = G.window.open(url, "GoogleAuthPopup", windowFeatures);
if (!authWindow) {
// This method should be invoked by an user action.
throw new Error("This method should be invoked synchronously");
}
return authWindow;
}

@ -344,6 +344,13 @@ export interface DocAPI {
// is HEAD, the result will contain a copy of any rows added or updated.
compareVersion(leftHash: string, rightHash: string): Promise<DocStateComparison>;
getDownloadUrl(template?: boolean): string;
/**
* Exports current document to the Google Drive as a spreadsheet file. To invoke this method, first
* acquire "code" via Google Auth Endpoint (see ShareMenu.ts for an example).
* @param code Authorization code returned from Google (requested via Grist's Google Auth Endpoint)
* @param title Name of the spreadsheet that will be created (should use a Grist document's title)
*/
sendToDrive(code: string, title: string): Promise<{url: string}>;
}
// Operations that are supported by a doc worker.
@ -775,4 +782,11 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
public getDownloadUrl(template: boolean = false) {
return this._url + `/download?template=${Number(template)}`;
}
public async sendToDrive(code: string, title: string): Promise<{url: string}> {
const url = new URL(`${this._url}/send-to-drive`);
url.searchParams.append('title', title);
url.searchParams.append('code', code);
return this.requestJson(url.href);
}
}

@ -133,7 +133,7 @@ export function undef<T extends Array<any>>(...list: T): Undef<T> {
*/
export function safeJsonParse(json: string, defaultVal: any): any {
try {
return JSON.parse(json);
return json !== '' && json !== undefined ? JSON.parse(json) : defaultVal;
} catch (e) {
return defaultVal;
}

@ -42,6 +42,7 @@ export class DocApiForwarder {
app.use('/api/docs/:docId/remove', withDoc);
app.delete('/api/docs/:docId', withDoc);
app.use('/api/docs/:docId/download', withDoc);
app.use('/api/docs/:docId/send-to-drive', withDoc);
app.use('/api/docs/:docId/fork', withDoc);
app.use('/api/docs/:docId/create-fork', withDoc);
app.use('/api/docs/:docId/apply', withDoc);

@ -28,6 +28,8 @@ import * as contentDisposition from 'content-disposition';
import { Application, NextFunction, Request, RequestHandler, Response } from "express";
import fetch from 'node-fetch';
import * as path from 'path';
import { exportToDrive } from "app/server/lib/GoogleExport";
import { googleAuthTokenMiddleware } from "app/server/lib/GoogleAuth";
// Cap on the number of requests that can be outstanding on a single document via the
// rest doc api. When this limit is exceeded, incoming requests receive an immediate
@ -92,6 +94,8 @@ export class DocWorkerApi {
const canEditMaybeRemoved = expressWrap(this._assertAccess.bind(this, 'editors', true));
// check document exists, don't check user access
const docExists = expressWrap(this._assertAccess.bind(this, null, false));
// converts google code to access token and adds it to request object
const decodeGoogleToken = expressWrap(googleAuthTokenMiddleware.bind(null));
// Middleware to limit number of outstanding requests per document. Will also
// handle errors like expressWrap would.
@ -443,6 +447,8 @@ export class DocWorkerApi {
res.json(result);
}));
this._app.get('/api/docs/:docId/send-to-drive', canView, decodeGoogleToken, withDoc(exportToDrive));
// Create a document. When an upload is included, it is imported as the initial
// state of the document. Otherwise a fresh empty document is created.
// A "timezone" option can be supplied.

@ -13,7 +13,7 @@ import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import * as log from 'app/server/lib/log';
import {integerParam, optStringParam, stringParam} from 'app/server/lib/requestUtils';
import {OpenMode, quoteIdent, SQLiteDB} from 'app/server/lib/SQLiteDB';
import {generateCSV} from 'app/server/serverMethods';
import {generateCSV, generateXLSX} from 'app/server/serverMethods';
import * as contentDisposition from 'content-disposition';
import * as express from 'express';
import * as fse from 'fs-extra';
@ -34,6 +34,10 @@ export class DocWorker {
await generateCSV(req, res, this._comm);
}
public async getXLSX(req: express.Request, res: express.Response): Promise<void> {
await generateXLSX(req, res, this._comm);
}
public async getAttachment(req: express.Request, res: express.Response): Promise<void> {
try {
const docSession = this._getDocSession(stringParam(req.query.clientId),

@ -0,0 +1,257 @@
import {CellValue} from 'app/common/DocActions';
import {GristType} from 'app/common/gristTypes';
import * as gutil from 'app/common/gutil';
import * as gristTypes from 'app/common/gristTypes';
import {NumberFormatOptions} from 'app/common/NumberFormat';
import {formatUnknown, IsRightTypeFunc} from 'app/common/ValueFormatter';
import {decodeObject} from 'app/plugin/objtypes';
import {Style} from 'exceljs';
import * as moment from 'moment-timezone';
interface WidgetOptions extends NumberFormatOptions {
textColor?: 'string';
fillColor?: 'string';
alignment?: 'left' | 'center' | 'right';
dateFormat?: string;
timeFormat?: string;
}
class BaseFormatter {
protected isRightType: IsRightTypeFunc;
protected widgetOptions: WidgetOptions = {};
constructor(public type: string, public opts: object) {
this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) ||
gristTypes.isRightType('Any')!;
this.widgetOptions = opts;
}
/**
* Formats a value that matches the type of this formatter. This should be overridden by derived
* classes to handle values in formatter-specific ways.
*/
public format(value: any): any {
return value;
}
public style(): Partial<Style> {
const argb = (hex: string) => `FF${hex.substr(1)}`;
const style: Partial<Style> = {};
if (this.widgetOptions.fillColor) {
style.fill = {
type: 'pattern',
pattern: 'solid',
fgColor: { argb: argb(this.widgetOptions.fillColor) }
};
}
if (this.widgetOptions.textColor) {
style.font = {
color: { argb: argb(this.widgetOptions.textColor) }
};
}
if (this.widgetOptions.alignment) {
style.alignment = {
horizontal: this.widgetOptions.alignment
};
}
if (this.widgetOptions.dateFormat) {
style.numFmt = excelDateFormat(this.widgetOptions.dateFormat, 'yyyy-mm-dd');
}
if (this.widgetOptions.timeFormat) {
style.numFmt = excelDateFormat(this.widgetOptions.dateFormat!, 'yyyy-mm-dd') + ' ' +
excelDateFormat(this.widgetOptions.timeFormat, 'h:mm am/pm');
}
// For number formats - we will support default excel formatting only,
// those formats strings are the defaults that LibreOffice Calc is using.
if (this.widgetOptions.numMode) {
if (this.widgetOptions.numMode === 'currency') {
style.numFmt = '[$$-409]#,##0.00';
} else if (this.widgetOptions.numMode === 'percent') {
style.numFmt = '0.00%';
} else if (this.widgetOptions.numMode === 'decimal') {
style.numFmt = '0.00';
} else if (this.widgetOptions.numMode === 'scientific') {
style.numFmt = '0.00E+00';
}
}
return style;
}
/**
* Formats using this.format() if a value is of the right type for this formatter, or using
* formatUnknown (like AnyFormatter) otherwise, resulting in a string representation.
*/
public formatAny(value: any): any {
return this.isRightType(value) ? this.format(value) : formatUnknown(value);
}
}
class AnyFormatter extends BaseFormatter {
public format(value: any): any {
return formatUnknown(value);
}
}
class ChoiceListFormatter extends BaseFormatter {
public format(value: any): any {
const obj = decodeObject(value);
if (Array.isArray(obj)) {
return obj.join("; ");
}
return formatUnknown(value);
}
}
class UnsupportedFormatter extends BaseFormatter {
public format(value: any): any {
return '';
}
}
class NumberFormatter extends BaseFormatter {
public format(value: any): any {
return Number.isFinite(value) ? value : '';
}
}
class DateFormatter extends BaseFormatter {
private _timezone: string;
constructor(type: string, opts: WidgetOptions, timezone: string = 'UTC') {
opts.dateFormat = opts.dateFormat || 'YYYY-MM-DD';
super(type, opts);
this._timezone = timezone || 'UTC';
// For native conversion - booleans are not a right type.
this.isRightType = (value: CellValue) => typeof value === 'number';
}
public format(value: any): any {
if (value === null) { return ''; }
// convert time to correct timezone
const time = moment(value * 1000).tz(this._timezone);
// in case moment is not able to interpret this as a valid date
// fallback to formatUnknown, for example for 0, NaN, Infinity
if (!time) {
return formatUnknown(value);
}
// make it look like a local time
time.utc(true).local();
// moment objects are mutable so we can just return original object.
return time.toDate();
}
}
class DateTimeFormatter extends DateFormatter {
constructor(type: string, opts: WidgetOptions) {
const timezone = gutil.removePrefix(type, "DateTime:") || '';
opts.timeFormat = opts.timeFormat === undefined ? 'h:mma' : opts.timeFormat;
super(type, opts, timezone);
}
}
const formatters: Partial<Record<GristType, typeof BaseFormatter>> = {
// for numbers - return javascript number
Numeric: NumberFormatter,
Int: NumberFormatter,
// for booleans - return javascript booleans
Bool: BaseFormatter,
// for dates - return javascript Date object
Date: DateFormatter,
DateTime: DateTimeFormatter,
ChoiceList: ChoiceListFormatter,
// for attachments - return blank cell
Attachments: UnsupportedFormatter,
// for anything else - return string (use default AnyFormatter)
};
/**
* Takes column type and widget options and returns a constructor with a format function that can
* properly convert a value passed to it into the right javascript object for that column.
* Exceljs library is using javascript primitives to specify correct excel type.
*/
export function createExcelFormatter(type: string, opts: object): BaseFormatter {
const ctor = formatters[gristTypes.extractTypeFromColType(type) as GristType] || AnyFormatter;
return new ctor(type, opts);
}
// ----------------------------------------------------------------------
// Helper functions
// ----------------------------------------------------------------------
// Mapping from moment-js basic date format tokens to excel numFmt basic tokens.
// We will convert all our predefined format to excel ones, and try to do our
// best on converting custom formats. If we fail on custom formats we will fall
// back to default ones.
// More on formats can be found:
// https://docs.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.numberingformats?view=openxml-2.8.1
// http://officeopenxml.com/WPdateTimeFieldSwitches.php
const mapping = new Map<string, string>();
mapping.set('YYYY', 'yyyy');
mapping.set('YY', 'yy');
mapping.set('M', 'm');
mapping.set('MM', 'mm');
mapping.set('MMM', 'mmm');
mapping.set('MMMM', 'mmmm');
mapping.set('D', 'd');
mapping.set('DD', 'dd');
mapping.set('DDD', 'ddd');
mapping.set('DDDD', 'dddd');
mapping.set('Do', 'dd'); // no direct match
mapping.set('L', 'yyyy-mm-dd');
mapping.set('LL', 'mmmmm d yyyy');
mapping.set('LLL', 'mmmmm d yyyy h:mm am/pm');
mapping.set('LLLL', 'ddd, mmmmm d yyyy h:mm am/pm');
mapping.set('h', 'h');
mapping.set('HH', 'hh');
// Minutes formats are the same as month's ones, but when they are after hour format
// they are treated as minutes.
mapping.set('m', 'm');
mapping.set('mm', 'mm');
mapping.set('mma', 'mm am/pm');
mapping.set('ss', 'ss');
mapping.set('s', 's');
mapping.set('a', 'am/pm');
mapping.set('A', 'am/pm');
mapping.set('S', '0');
mapping.set('SS', '00');
mapping.set('SSS', '000');
mapping.set('SSSS', '0000');
mapping.set('SSSSS', '00000');
mapping.set('SSSSSS', '000000');
// We will omit timezone formats
mapping.set('z', '');
mapping.set('zz', '');
mapping.set('Z', '');
mapping.set('ZZ', '');
/**
* Converts Moment js format string to excel numFormat
* @param format Moment js format string
* @param def Default excel format string
*/
function excelDateFormat(format: string, def: string) {
// split format to chunks by common separator
const chunks = format.split(/([\s:.,-/]+)/);
// try to map chunks
for (let i = 0; i < chunks.length; i += 2) {
const chunk = chunks[i];
if (mapping.has(chunk)) {
chunks[i] = mapping.get(chunk)!;
} else {
// fail on first mismatch
return def;
}
}
// fix the separators - they need to be prefixed by backslash
for (let i = 1; i < chunks.length; i += 2) {
const sep = chunks[i];
if (sep === '-') {
chunks[i] = '\\-';
}
if (sep.trim() === '') {
chunks[i] = '\\' + sep;
}
}
return chunks.join('');
}

@ -0,0 +1,299 @@
import {buildColFilter} from 'app/common/ColumnFilterFunc';
import {RowRecord} from 'app/common/DocActions';
import {DocData} from 'app/common/DocData';
import * as gristTypes from 'app/common/gristTypes';
import * as gutil from 'app/common/gutil';
import {buildRowFilter} from 'app/common/RowFilterFunc';
import {SchemaTypes} from 'app/common/schema';
import {SortFunc} from 'app/common/SortFunc';
import {TableData} from 'app/common/TableData';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {RequestWithLogin} from 'app/server/lib/Authorizer';
import {docSessionFromRequest} from 'app/server/lib/DocSession';
import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters';
import * as express from 'express';
import * as _ from 'underscore';
// Helper type for Cell Accessor
type Access = (row: number) => any;
// Helper interface with information about the column
interface ExportColumn {
id: number;
colId: string;
label: string;
type: string;
widgetOptions: any;
parentPos: number;
}
// helper for empty column
const emptyCol: ExportColumn = {
id: 0,
colId: '',
label: '',
type: '',
widgetOptions: null,
parentPos: 0
};
/**
* Bare data that is exported - used to convert to various formats.
*/
export interface ExportData {
/**
* Table name or table id.
*/
tableName: string;
/**
* Document name.
*/
docName: string;
/**
* Row ids (filtered and sorted).
*/
rowIds: number[];
/**
* Accessor for value in a column.
*/
access: Access[];
/**
* Columns information (primary used for formatting).
*/
columns: ExportColumn[];
}
/**
* Export parameters that identifies a section, filters, sort order.
*/
export interface ExportParameters {
tableId: string;
viewSectionId: number;
sortOrder: number[];
filters: Filter[]
}
/**
* Gets export parameters from a request.
*/
export function parseExportParameters(req: express.Request): ExportParameters {
const tableId = req.query.tableId;
const viewSectionId = parseInt(req.query.viewSection, 10);
const sortOrder = gutil.safeJsonParse(req.query.activeSortSpec, null) as number[];
const filters: Filter[] = gutil.safeJsonParse(req.query.filters, []) || [];
return {
tableId,
viewSectionId,
sortOrder,
filters
};
}
/**
* Calculates the file name (without an extension) for exported table.
* @param activeDoc ActiveDoc
* @param req Request (with export params)
*/
export function parseExportFileName(activeDoc: ActiveDoc, req: express.Request) {
const title = req.query.title;
const tableId = req.query.tableId;
const docName = title || activeDoc.docName;
const name = docName +
(tableId === docName ? '' : '-' + tableId);
return name;
}
// Makes assertion that value does exists or throws an error
function safe<T>(value: T, msg: string) {
if (!value) { throw new Error(msg); }
return value as NonNullable<T>;
}
// Helper to for getting table from docData.
const safeTable = (docData: DocData, name: keyof SchemaTypes) => safe(docData.getTable(name),
`No table '${name}' in document with id ${docData}`);
// Helper for getting record safe
const safeRecord = (table: TableData, id: number) => safe(table.getRecord(id),
`No record ${id} in table ${table.tableId}`);
/**
* Builds export for all raw tables that are in doc.
* @param activeDoc Active document
* @param req Request
*/
export async function exportDoc(
activeDoc: ActiveDoc,
req: express.Request) {
const docData = safe(activeDoc.docData, "No docData in active document");
const tables = safeTable(docData, '_grist_Tables');
// select raw tables
const tableIds = tables.filterRowIds({ summarySourceTable: 0 });
const tableExports = await Promise.all(
tableIds
.map(tId => exportTable(activeDoc, tId, req))
);
return tableExports;
}
/**
* Builds export data for section that can be used to produce files in various formats (csv, xlsx).
*/
export async function exportTable(
activeDoc: ActiveDoc,
tableId: number,
req: express.Request): Promise<ExportData> {
const docData = safe(activeDoc.docData, "No docData in active document");
const tables = safeTable(docData, '_grist_Tables');
const table = safeRecord(tables, tableId) as GristTables;
const tableColumns = (safeTable(docData, '_grist_Tables_column')
.getRecords() as GristTablesColumn[])
// remove manual sort column
.filter(col => col.colId !== gristTypes.MANUALSORT);
// Produce a column description matching what user will see / expect to export
const tableColsById = _.indexBy(tableColumns, 'id');
const columns = tableColumns.map(tc => {
// remove all columns that don't belong to this table
if (tc.parentId !== tableId) {
return emptyCol;
}
// remove all helpers
if (gristTypes.isHiddenCol(tc.colId)) {
return emptyCol;
}
// for reference columns, return display column, and copy settings from visible column
const displayCol = tableColsById[tc.displayCol || tc.id];
const colOptions = gutil.safeJsonParse(tc.widgetOptions, {});
const displayOptions = gutil.safeJsonParse(displayCol.widgetOptions, {});
const widgetOptions = Object.assign(displayOptions, colOptions);
return {
id: displayCol.id,
colId: displayCol.colId,
label: tc.label,
type: displayCol.type,
widgetOptions,
parentPos: tc.parentPos
};
}).filter(tc => tc !== emptyCol);
// fetch actual data
const data = await activeDoc.fetchTable(docSessionFromRequest(req as RequestWithLogin), table.tableId, true);
const rowIds = data[2];
const dataByColId = data[3];
// sort rows
const getters = new ServerColumnGetters(rowIds, dataByColId, columns);
// create cell accessors
const access = columns.map(col => getters.getColGetter(col.id)!);
let tableName = table.tableId;
// since tables ids are not very friendly, borrow name from a primary view
if (table.primaryViewId) {
const viewId = table.primaryViewId;
const views = safeTable(docData, '_grist_Views');
const view = safeRecord(views, viewId) as GristView;
tableName = view.name;
}
return {
tableName,
docName: activeDoc.docName,
rowIds,
access,
columns
};
}
/**
* Builds export data for section that can be used to produce files in various formats (csv, xlsx).
*/
export async function exportSection(
activeDoc: ActiveDoc,
viewSectionId: number,
sortOrder: number[] | null,
filters: Filter[] | null,
req: express.Request): Promise<ExportData> {
const docData = safe(activeDoc.docData, "No docData in active document");
const viewSections = safeTable(docData, '_grist_Views_section');
const viewSection = safeRecord(viewSections, viewSectionId) as GristViewsSection;
const tables = safeTable(docData, '_grist_Tables');
const table = safeRecord(tables, viewSection.tableRef) as GristTables;
const columns = safeTable(docData, '_grist_Tables_column')
.filterRecords({ parentId: table.id }) as GristTablesColumn[];
const viewSectionFields = safeTable(docData, '_grist_Views_section_field');
const fields = viewSectionFields.filterRecords({ parentId: viewSection.id }) as GristViewsSectionField[];
const tableColsById = _.indexBy(columns, 'id');
// Produce a column description matching what user will see / expect to export
const viewify = (col: GristTablesColumn, field: GristViewsSectionField) => {
field = field || {};
const displayCol = tableColsById[field.displayCol || col.displayCol || col.id];
const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {});
const fieldWidgetOptions = gutil.safeJsonParse(field.widgetOptions, {});
const filterString = (filters || []).find(x => x.colRef === field.colRef)?.filter || field.filter;
const filterFunc = buildColFilter(filterString, col.type);
return {
id: displayCol.id,
colId: displayCol.colId,
label: col.label,
type: col.type,
parentPos: col.parentPos,
filterFunc,
widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions)
};
};
const viewColumns = _.sortBy(fields, 'parentPos').map(
(field) => viewify(tableColsById[field.colRef], field));
// The columns named in sort order need to now become display columns
sortOrder = sortOrder || gutil.safeJsonParse(viewSection.sortColRefs, []);
const fieldsByColRef = _.indexBy(fields, 'colRef');
sortOrder = sortOrder!.map((directionalColRef) => {
const colRef = Math.abs(directionalColRef);
const col = tableColsById[colRef];
if (!col) {
return 0;
}
const effectiveColRef = viewify(col, fieldsByColRef[colRef]).id;
return directionalColRef > 0 ? effectiveColRef : -effectiveColRef;
});
// fetch actual data
const data = await activeDoc.fetchTable(docSessionFromRequest(req as RequestWithLogin), table.tableId, true);
let rowIds = data[2];
const dataByColId = data[3];
// sort rows
const getters = new ServerColumnGetters(rowIds, dataByColId, columns);
const sorter = new SortFunc(getters);
sorter.updateSpec(sortOrder);
rowIds.sort((a, b) => sorter.compare(a, b));
// create cell accessors
const access = viewColumns.map(col => getters.getColGetter(col.id)!);
// create row filter based on all columns filter
const rowFilter = viewColumns
.map((col, c) => buildRowFilter(access[c], col.filterFunc))
.reduce((prevFilter, curFilter) => (id) => prevFilter(id) && curFilter(id), () => true);
// filter rows numbers
rowIds = rowIds.filter(rowFilter);
return {
tableName: table.tableId,
docName: activeDoc.docName,
rowIds,
access,
columns: viewColumns
};
}
// Type helpers for types used in this export
type RowModel<TName extends keyof SchemaTypes> = RowRecord & {
[ColId in keyof SchemaTypes[TName]]: SchemaTypes[TName][ColId];
};
type GristViewsSection = RowModel<'_grist_Views_section'>
type GristTables = RowModel<'_grist_Tables'>
type GristViewsSectionField = RowModel<'_grist_Views_section_field'>
type GristTablesColumn = RowModel<'_grist_Tables_column'>
type GristView = RowModel<'_grist_Views'>
// Type for filters passed from the client
export interface Filter { colRef: number, filter: string }

@ -0,0 +1,48 @@
import {createFormatter} from 'app/common/ValueFormatter';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {ExportData, exportSection, Filter} from 'app/server/lib/Export';
import * as bluebird from 'bluebird';
import * as csv from 'csv';
import * as express from 'express';
// promisify csv
bluebird.promisifyAll(csv);
/**
* Returns a csv stream that can be transformed or parsed. See https://github.com/wdavidw/node-csv
* for API details.
*
* @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
* @param {Integer} viewSectionId - id of the viewsection to export.
* @param {Integer[]} activeSortOrder (optional) - overriding sort order.
* @param {Filter[]} filters (optional) - filters defined from ui.
* @return {Promise<string>} Promise for the resulting CSV.
*/
export async function makeCSV(
activeDoc: ActiveDoc,
viewSectionId: number,
sortOrder: number[],
filters: Filter[],
req: express.Request) {
const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, req);
const file = convertToCsv(data);
return file;
}
function convertToCsv({
rowIds,
access,
columns: viewColumns
}: ExportData) {
// create formatters for columns
const formatters = viewColumns.map(col => createFormatter(col.type, col.widgetOptions));
// Arrange the data into a row-indexed matrix, starting with column headers.
const csvMatrix = [viewColumns.map(col => col.label)];
// populate all the rows with values as strings
rowIds.forEach(row => {
csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter(row))));
});
return csv.stringifyAsync(csvMatrix);
}

@ -0,0 +1,91 @@
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {createExcelFormatter} from 'app/server/lib/ExcelFormatter';
import {ExportData, exportDoc} from 'app/server/lib/Export';
import {Alignment, Border, Fill, Workbook} from 'exceljs';
import * as express from 'express';
/**
* Creates excel document with all tables from an active Grist document.
*/
export async function makeXLSX(
activeDoc: ActiveDoc,
req: express.Request): Promise<ArrayBuffer> {
const content = await exportDoc(activeDoc, req);
const data = await convertToExcel(content, req.host === 'localhost');
return data;
}
/**
* Converts export data to an excel file.
*/
async function convertToExcel(tables: ExportData[], testDates: boolean) {
// Create workbook and add single sheet to it.
const wb = new Workbook();
if (testDates) {
// HACK: for testing, we will keep static dates
const date = new Date(Date.UTC(2018, 11, 1, 0, 0, 0));
wb.modified = date;
wb.created = date;
wb.lastPrinted = date;
wb.creator = 'test';
wb.lastModifiedBy = 'test';
}
// Prepare border - some of the cells can have background colors, in that case border will
// not be visible
const borderStyle: Border = {
color: { argb: 'FFE2E2E3' }, // dark gray - default border color for gdrive
style: 'thin'
};
const borders = {
left: borderStyle,
right: borderStyle,
top: borderStyle,
bottom: borderStyle
};
const headerBackground: Fill = {
type: 'pattern',
pattern: 'solid',
fgColor: { argb: 'FFEEEEEE' } // gray
};
const headerFontColor = {
color: {
argb: 'FF000000' // black
}
};
const centerAlignment: Partial<Alignment> = {
horizontal: 'center'
};
for (const table of tables) {
const { columns, rowIds, access, tableName } = table;
const ws = wb.addWorksheet(tableName);
// Build excel formatters.
const formatters = columns.map(col => createExcelFormatter(col.type, col.widgetOptions));
// Generate headers for all columns with correct styles for whole column.
// Actual header style for a first row will be overwritten later.
ws.columns = columns.map((col, c) => ({ header: col.label, style: formatters[c].style() }));
// Populate excel file with data
rowIds.forEach(row => {
ws.addRow(access.map((getter, c) => formatters[c].formatAny(getter(row))));
});
// style up the header row
for (let i = 1; i <= columns.length; i++) {
// apply to all rows (including header)
ws.getColumn(i).border = borders;
// apply only to header
const header = ws.getCell(1, i);
header.fill = headerBackground;
header.font = headerFontColor;
header.alignment = centerAlignment;
}
// Make each column a little wider.
ws.columns.forEach(column => {
if (!column.header) {
return;
}
// 14 points is about 100 pixels in a default font (point is around 7.5 pixels)
column.width = column.header.length < 14 ? 14 : column.header.length;
});
}
return await wb.xlsx.writeBuffer();
}

@ -45,7 +45,7 @@ import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint'
import {PluginManager} from 'app/server/lib/PluginManager';
import {adaptServerUrl, addOrgToPath, addPermit, getScope, optStringParam, RequestWithGristInfo, stringParam,
TEST_HTTPS_OFFSET, trustOrigin} from 'app/server/lib/requestUtils';
import {ISendAppPageOptions, makeGristConfig, makeSendAppPage} from 'app/server/lib/sendAppPage';
import {ISendAppPageOptions, makeGristConfig, makeMessagePage, makeSendAppPage} from 'app/server/lib/sendAppPage';
import {getDatabaseUrl} from 'app/server/lib/serverUtils';
import {Sessions} from 'app/server/lib/Sessions';
import * as shutdown from 'app/server/lib/shutdown';
@ -64,6 +64,7 @@ import {AddressInfo} from 'net';
import fetch from 'node-fetch';
import * as path from 'path';
import * as serveStatic from "serve-static";
import { addGoogleAuthEndpoint } from "app/server/lib/GoogleAuth";
// Health checks are a little noisy in the logs, so we don't show them all.
// We show the first N health checks:
@ -1229,6 +1230,12 @@ export class FlexServer implements GristServer {
}
}
public addGoogleAuthEndpoint() {
if (this._check('google-auth')) { return; }
const messagePage = makeMessagePage(this, getAppPathTo(this.appRoot, 'static'));
addGoogleAuthEndpoint(this.app, messagePage);
}
// Adds endpoints that support imports and exports.
private _addSupportPaths(docAccessMiddleware: express.RequestHandler[]) {
if (!this._docWorker) { throw new Error("need DocWorker"); }
@ -1252,6 +1259,10 @@ export class FlexServer implements GristServer {
return this._docWorker.getCSV(req, res);
}));
this.app.get('/gen_xlsx', ...docAccessMiddleware, expressWrap(async (req, res) => {
return this._docWorker.getXLSX(req, res);
}));
this.app.get('/attachment', ...docAccessMiddleware,
expressWrap(async (req, res) => this._docWorker.getAttachment(req, res)));
}

@ -0,0 +1,170 @@
import {auth} from '@googleapis/oauth2';
import {ApiError} from 'app/common/ApiError';
import {parseSubdomain} from 'app/common/gristUrls';
import {expressWrap} from 'app/server/lib/expressWrap';
import * as log from 'app/server/lib/log';
import * as express from 'express';
import {URL} from 'url';
/**
* Google Auth Endpoint for performing server side authentication. More information can be found
* at https://developers.google.com/identity/protocols/oauth2/web-server.
*
* Environmental variables used:
* - GOOGLE_CLIENT_ID : key obtained from a Google Project, not secret can be shared publicly,
* the same client id is used in Google Drive Plugin
* - GOOGLE_CLIENT_SECRET: secret key for Google Project, can't be shared - it is used to (d)encrypt
* data that we obtain from Google Auth Service (all done in the api)
*
* High level description:
*
* Each API endpoint that wants to talk to Google Api needs an access_token that identifies our application
* and permits us to access some of the user data or work with the API on the user's behalf (examples for that are
* Google Drive plugin and Google Export endpoint, [Send to Google Drive] feature on the UI).)
* To obtain this token on the server-side, the application needs to redirect the user to a
* Google Consent Screen - where the user can log into his account and give consent for the permissions
* we need. Permissions are defined by SCOPES - that exactly describes what we are allowed to do.
* More on scopes can be read on https://developers.google.com/identity/protocols/oauth2/scopes.
* When we are redirecting the user to a Google Consent Screen, we are also sending a static URL for an endpoint
* where Google will redirect the user after he gives us permissions or declines our request. For that, we are exposing
* static URL http://docs.getgrist.com/auth/google (on prod) or http://localhost:8080/auth/google (dev).
*
* NOTE: Actually, we are exposing only auth/google endpoint that can be accessed in various ways, including any
* subdomain, but Google will always redirect to the configured endpoint (example: http://docs.getgrist.com/auth/google)
*
* This endpoint will render a simple page (see /static/message.html) that will immediately post
* a message to the parent window with a response from Google in the form of { code? : string, error?: string }.
* Code returned from Google will be an encrypted access_token that the client-side code should use to invoke
* the server-side API endpoint that wishes to call one of the Google API endpoints. A server-side endpoint can use
* "googleAuthTokenMiddleware" middleware to convert code to access_token (by making a separate call to Google
* for exchanging code for an access_token).
* This access_token could be stored in the user's session for further use, but since we are making only a single call
* very rarely, and access_token will expire eventually; it is better to acquire access_token every time.
* More on storing access_token offline can be read on:
* https://developers.google.com/identity/protocols/oauth2/web-server#obtainingaccesstokens
*
* How to use:
*
* To call server-side endpoint that expects access_token, first decorate it with "googleAuthTokenMiddleware"
* middleware, then perform "server-side" authentication with Google on the client-side to acquire an encrypted token.
* Client code should open up a popup window with an URL to Grist's Google Auth endpoint (/auth/google) and wait
* for a message from a popup window containing an encrypted token.
* Having encrypted token ("code"), a client can call the server-side endpoint by adding to the query string the code
* acquired from the popup window. Server endpoint (decorated by "googleAuthTokenMiddleware") will get access_token
* in a query string.
*/
// Path for the auth endpoint.
const authHandlerPath = "/auth/google";
// "View and manage Google Drive files and folders that you have opened or created with this app.""
export const DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.file';
// Redirect host after the Google Auth login form is completed. This reuses the same domain name
// as for Cognito login.
const AUTH_SUBDOMAIN = process.env.GRIST_ID_PREFIX ? `docs-${process.env.GRIST_ID_PREFIX}` : 'docs';
/**
* Return a full url for Google Auth handler. Examples are:
*
* http://localhost:8080/auth/google in dev
* https://docs-s.getgrist.com in staging
* https://docs.getgrist.com in prod
*/
function getFullAuthEndpointUrl(): string {
const homeUrl = process.env.APP_HOME_URL;
// if homeUrl is localhost - (in dev environment) - use the development url
if (homeUrl && new URL(homeUrl).hostname === "localhost") {
return `${homeUrl}${authHandlerPath}`;
}
const homeBaseDomain = homeUrl && parseSubdomain(new URL(homeUrl).host).base;
const baseDomain = homeBaseDomain || '.getgrist.com';
return `https://${AUTH_SUBDOMAIN}${baseDomain}${authHandlerPath}`;
}
/**
* Middleware for obtaining access_token from Google Auth Service.
* It expects code query parameter (provided by frontend code) add adds to access_token to query parameters.
*/
export async function googleAuthTokenMiddleware(
req: express.Request,
res: express.Response,
next: express.NextFunction) {
// If access token is in place, proceed
if (!req.query.code) {
throw new ApiError("Google Auth endpoint requires a code parameter in the query string", 400);
} else {
try {
const oAuth2Client = _googleAuthClient();
// Decrypt code that was send back from Google Auth service. Uses GOOGLE_CLIENT_SECRET key.
const tokenResponse = await oAuth2Client.getToken(req.query.code);
// Get the access token (access token will be present in a default request configuration).
const access_token = tokenResponse.tokens.access_token!;
req.query.access_token = access_token;
next();
} catch (err) {
log.error("GoogleAuth - Error", err);
throw err;
}
}
}
/**
* Adds a static Google Auth endpoint. This will be used by Google Auth Service to redirect back, after the user
* finishes a signing and a consent flow. Google will pass 2 arguments in a query string:
* - code: encrypted access token when user gave permissions
* - error: error code when user declined our request.
*/
export function addGoogleAuthEndpoint(
expressApp: express.Application,
messagePage: (req: express.Request, res: express.Response, message: any) => any
) {
if (!process.env.GOOGLE_CLIENT_SECRET) {
log.error("Failed to create GoogleAuth endpoint: GOOGLE_CLIENT_SECRET is not defined");
expressApp.get(authHandlerPath, expressWrap(async (req: express.Request, res: express.Response) => {
throw new Error("Send to Google Drive is not configured.");
}));
return;
}
log.info(`GoogleAuth - auth handler at ${getFullAuthEndpointUrl()}`);
expressApp.get(authHandlerPath, expressWrap(async (req: express.Request, res: express.Response) => {
// Test if the code is in a query string. Google sends it back after user has given a concent for
// our request. It is encrypted (with CLIENT_SECRET) and signed with redirect url.
if (req.query.code) {
log.debug("GoogleAuth - response from Google with valid code");
messagePage(req, res, { code: req.query.code });
} else if (req.query.error) {
log.debug("GoogleAuth - response from Google with error code", req.query.error);
if (req.query.error === "access_denied") {
messagePage(req, res, { error: req.query.error });
} else {
// This should not happen, either code or error is a mandatory query parameter.
throw new ApiError("Error authenticating with Google", 500);
}
} else {
const oAuth2Client = _googleAuthClient();
const scope = req.query.scope || DRIVE_SCOPE;
const authUrl = oAuth2Client.generateAuthUrl({
scope,
prompt: 'select_account'
});
log.debug(`GoogleAuth - redirecting to Google consent screen`, {
authUrl,
scope
});
res.redirect(authUrl);
}
}));
}
/**
* Builds the OAuth2 Google client.
*/
function _googleAuthClient() {
const CLIENT_ID = process.env.GOOGLE_CLIENT_ID;
const CLIENT_SECRET = process.env.GOOGLE_CLIENT_SECRET;
const oAuth2Client = new auth.OAuth2(CLIENT_ID, CLIENT_SECRET, getFullAuthEndpointUrl());
return oAuth2Client;
}

@ -0,0 +1,83 @@
import {drive} from '@googleapis/drive';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {RequestWithLogin} from 'app/server/lib/Authorizer';
import {makeXLSX} from 'app/server/lib/ExportXLSX';
import * as log from 'app/server/lib/log';
import {Request, Response} from 'express';
import {PassThrough} from 'stream';
/**
* Endpoint logic for sending grist document to Google Drive. Grist document is first exported as an
* excel file and then pushed to Google Drive as a Google Spreadsheet.
*/
export async function exportToDrive(
activeDoc: ActiveDoc,
req: Request,
res: Response
) {
// Token should come from auth middleware
const access_token = req.query.access_token;
if (!access_token) {
throw new Error("No access token - Can't send file to Google Drive");
}
const meta = {
docId : activeDoc.docName,
userId : (req as RequestWithLogin).userId
};
// Prepare file for exporting.
log.debug(`Export to drive - Preparing file for export`, meta);
const { name, data } = await prepareFile(activeDoc, req);
try {
// Send file to GDrive and get the url for a preview.
const url = await sendFileToDrive(name, data, access_token);
log.debug(`Export to drive - File exported, redirecting to Google Spreadsheet ${url}`, meta);
res.json({ url });
} catch (err) {
log.error("Export to drive - Error while sending file to GDrive", meta, err);
// Test if google returned a valid error message.
if (err.errors && err.errors.length) {
throw new Error(err.errors[0].message);
} else {
throw err;
}
}
}
// Creates spreadsheet file in a Google drive, by sending an excel and requesting for conversion.
async function sendFileToDrive(fileNameNoExt: string, data: ArrayBuffer, oauth_token: string): Promise<string> {
// Here we are asking google drive to convert excel file to a google spreadsheet
const requestBody = {
// name of the spreadsheet to create
name: fileNameNoExt,
// mime type of the google spreadsheet
mimeType: 'application/vnd.google-apps.spreadsheet'
};
// wrap buffer into a stream
const stream = new PassThrough();
stream.end(data);
// Define what gets send - excel file
const media = {
mimeType: 'application/vnd.ms-excel',
body: stream
};
const googleDrive = drive("v3");
const fileRes = await googleDrive.files.create({
requestBody, // what to do with file - convert to spreadsheet
oauth_token, // access token
media, // file
fields: "webViewLink" // return webViewLink after creating file
});
const url = fileRes.data.webViewLink;
if (!url) {
throw new Error("Google Api has not returned valid response");
}
return url;
}
// Makes excel file the same way as export to excel works.
async function prepareFile(doc: ActiveDoc, req: Request) {
const data = await makeXLSX(doc, req);
const name = (req.query.title || doc.docName);
return { name, data };
}

@ -46,6 +46,22 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadCo
};
}
/**
* Creates a method that will send html page that will immediately post a message to a parent window.
* Primary used for Google Auth Grist's endpoint, but can be used in future in any other server side
* authentication flow.
*/
export function makeMessagePage(server: GristServer, staticDir: string) {
return async (req: express.Request, resp: express.Response, message: any) => {
const config = server.getGristConfig();
const fileContent = await fse.readFile(path.join(staticDir, "message.html"), 'utf8');
const content = fileContent
.replace("<!-- INSERT CONFIG -->", `<script>window.gristConfig = ${JSON.stringify(config)};</script>`)
.replace("<!-- INSERT MESSAGE -->", `<script>window.message = ${JSON.stringify(message)};</script>`);
resp.status(200).type('html').send(content);
};
}
/**
* Send a simple template page, read from file at pagePath (relative to static/), with certain
* placeholders replaced.

@ -121,6 +121,7 @@ export async function main(port: number, serverTypes: ServerType[],
server.addBillingPages();
server.addWelcomePaths();
server.addLogEndpoint();
server.addGoogleAuthEndpoint();
}
if (includeDocs) {

@ -1,54 +1,34 @@
import * as gutil from 'app/common/gutil';
import { SortFunc } from "app/common/SortFunc";
import { docSessionFromRequest } from "app/server/lib/DocSession";
import * as bluebird from "bluebird";
import * as contentDisposition from "content-disposition";
import * as csv from "csv";
import * as log from "./lib/log";
import { ServerColumnGetters } from "./lib/ServerColumnGetters";
import * as _ from "underscore";
import * as express from "express";
import * as Comm from 'app/server/lib/Comm';
import { ActiveDoc } from "app/server/lib/ActiveDoc";
import { createFormatter } from "app/common/ValueFormatter";
import { SchemaTypes } from "app/common/schema";
import { RequestWithLogin } from "app/server/lib/Authorizer";
import { RowRecord } from "app/common/DocActions";
import { buildColFilter } from "app/common/ColumnFilterFunc";
import { buildRowFilter } from "app/common/RowFilterFunc";
// promisify csv
bluebird.promisifyAll(csv);
import {parseExportFileName, parseExportParameters} from 'app/server/lib/Export';
import {makeCSV} from 'app/server/lib/ExportCSV';
import {makeXLSX} from 'app/server/lib/ExportXLSX';
import * as log from 'app/server/lib/log';
import * as contentDisposition from 'content-disposition';
import * as express from 'express';
export async function generateCSV(req: express.Request, res: express.Response, comm: Comm) {
log.info('Generating .csv file...');
// Get the current table id
const tableId = req.param('tableId');
const viewSectionId = parseInt(req.param('viewSection'), 10);
const activeSortOrder = gutil.safeJsonParse(req.param('activeSortSpec'), null);
const filters: Filter[] = gutil.safeJsonParse(req.param("filters"), []) || [];
const {
viewSectionId,
filters,
sortOrder
} = parseExportParameters(req);
// Get the active doc
const clientId = req.param('clientId');
const docFD = parseInt(req.param('docFD'), 10);
const clientId = req.query.clientId;
const docFD = parseInt(req.query.docFD, 10);
const client = comm.getClient(clientId);
const docSession = client.getDocSession(docFD);
const activeDoc = docSession.activeDoc;
// Generate a decent name for the exported file.
const docName = req.query.title || activeDoc.docName;
const name = docName +
(tableId === docName ? '' : '-' + tableId) + '.csv';
const name = parseExportFileName(activeDoc, req);
try {
const data = await makeCSV(activeDoc, viewSectionId, activeSortOrder, filters, req);
const data = await makeCSV(activeDoc, viewSectionId, sortOrder, filters, req);
res.set('Content-Type', 'text/csv');
res.setHeader('Content-Disposition', contentDisposition(name));
res.setHeader('Content-Disposition', contentDisposition(name + '.csv'));
res.send(data);
} catch (err) {
log.error("Exporting to CSV has failed. Request url: %s", req.url, err);
// send a generic information to client
const errHtml =
`<!doctype html>
<html>
@ -59,151 +39,28 @@ export async function generateCSV(req: express.Request, res: express.Response, c
}
}
/**
* Returns a csv stream that can be transformed or parsed. See https://github.com/wdavidw/node-csv
* for API details.
*
* @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
* @param {Integer} viewSectionId - id of the viewsection to export.
* @param {Integer[]} activeSortOrder (optional) - overriding sort order.
* @return {Promise<string>} Promise for the resulting CSV.
*/
export async function makeCSV(
activeDoc: ActiveDoc,
viewSectionId: number,
sortOrder: number[],
filters: Filter[],
req: express.Request) {
const {
table,
viewSection,
tableColumns,
fields
} = explodeSafe(activeDoc, viewSectionId);
const tableColsById = _.indexBy(tableColumns, 'id');
// Produce a column description matching what user will see / expect to export
const viewify = (col: GristTablesColumn, field: GristViewsSectionField) => {
field = field || {};
const displayCol = tableColsById[field.displayCol || col.displayCol || col.id];
const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {});
const fieldWidgetOptions = gutil.safeJsonParse(field.widgetOptions, {});
const filterFunc = buildColFilter(filters.find(x => x.colRef === field.colRef)?.filter, col.type);
return {
id: displayCol.id,
colId: displayCol.colId,
label: col.label,
colType: col.type,
filterFunc,
widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions)
};
};
const viewColumns = _.sortBy(fields, 'parentPos').map(
(field) => viewify(tableColsById[field.colRef], field));
// The columns named in sort order need to now become display columns
sortOrder = sortOrder || gutil.safeJsonParse(viewSection.sortColRefs, []);
const fieldsByColRef = _.indexBy(fields, 'colRef');
sortOrder = sortOrder.map((directionalColRef) => {
const colRef = Math.abs(directionalColRef);
const col = tableColsById[colRef];
if (!col) {
return 0;
}
const effectiveColRef = viewify(col, fieldsByColRef[colRef]).id;
return directionalColRef > 0 ? effectiveColRef : -effectiveColRef;
});
const data = await activeDoc.fetchTable(docSessionFromRequest(req as RequestWithLogin), table.tableId, true);
const rowIds = data[2];
const dataByColId = data[3];
const getters = new ServerColumnGetters(rowIds, dataByColId, tableColumns);
const sorter = new SortFunc(getters);
sorter.updateSpec(sortOrder);
rowIds.sort((a, b) => sorter.compare(a, b));
const formatters = viewColumns.map(col =>
createFormatter(col.colType, col.widgetOptions));
// Arrange the data into a row-indexed matrix, starting with column headers.
const csvMatrix = [viewColumns.map(col => col.label)];
const access = viewColumns.map(col => getters.getColGetter(col.id));
// create row filter based on all columns filter
const rowFilter = viewColumns
.map((col, c) => buildRowFilter(access[c], col.filterFunc))
.reduce((prevFilter, curFilter) => (id) => prevFilter(id) && curFilter(id), () => true);
rowIds.forEach(row => {
if (!rowFilter(row)) {
return;
}
csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter!(row))));
});
return csv.stringifyAsync(csvMatrix);
}
// helper method that retrieves various parts about view section
// from ActiveDoc
function explodeSafe(activeDoc: ActiveDoc, viewSectionId: number) {
const docData = activeDoc.docData;
if (!docData) {
// Should not happen unless there's a logic error
// This method is exported (for testing) so it is possible
// to call it without loading active doc first
throw new Error("Document hasn't been loaded yet");
}
const viewSection = docData
.getTable('_grist_Views_section')
?.getRecord(viewSectionId) as GristViewsSection | undefined;
if (!viewSection) {
throw new Error(`No table '_grist_Views_section' in document with id ${activeDoc.docName}`);
}
const table = docData
.getTable('_grist_Tables')
?.getRecord(viewSection.tableRef) as GristTables | undefined;
if (!table) {
throw new Error(`No table '_grist_Tables' in document with id ${activeDoc.docName}`);
}
const fields = docData
.getTable('_grist_Views_section_field')
?.filterRecords({ parentId: viewSection.id }) as GristViewsSectionField[] | undefined;
if (!fields) {
throw new Error(`No table '_grist_Views_section_field' in document with id ${activeDoc.docName}`);
}
const tableColumns = docData
.getTable('_grist_Tables_column')
?.filterRecords({ parentId: table.id }) as GristTablesColumn[] | undefined;
if (!tableColumns) {
throw new Error(`No table '_grist_Tables_column' in document with id ${activeDoc.docName}`);
export async function generateXLSX(req: express.Request, res: express.Response, comm: Comm) {
log.debug(`Generating .xlsx file`);
const clientId = req.query.clientId;
const docFD = parseInt(req.query.docFD, 10);
const client = comm.getClient(clientId);
const docSession = client.getDocSession(docFD);
const activeDoc = docSession.activeDoc;
try {
const data = await makeXLSX(activeDoc, req);
res.set('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
res.setHeader('Content-Disposition', contentDisposition((req.query.title || activeDoc.docName) + '.xlsx'));
res.send(data);
log.debug('XLSX file generated');
} catch (err) {
log.error("Exporting to XLSX has failed. Request url: %s", req.url, err);
// send a generic information to client
const errHtml =
`<!doctype html>
<html>
<body>There was an unexpected error while generating a xlsx file.</body>
</html>
`;
res.status(400).send(errHtml);
}
return {
table,
fields,
tableColumns,
viewSection
};
}
// Type helpers for types used in this export
type RowModel<TName extends keyof SchemaTypes> = RowRecord & {
[ColId in keyof SchemaTypes[TName]]: SchemaTypes[TName][ColId];
};
type GristViewsSection = RowModel<'_grist_Views_section'>
type GristTables = RowModel<'_grist_Tables'>
type GristViewsSectionField = RowModel<'_grist_Views_section_field'>
type GristTablesColumn = RowModel<'_grist_Tables_column'>
// Type for filters passed from the client
interface Filter { colRef: number, filter: string }

@ -65,6 +65,8 @@
"why-is-node-running": "2.0.3"
},
"dependencies": {
"@googleapis/drive": "0.3.1",
"@googleapis/oauth2": "0.2.0",
"@gristlabs/connect-sqlite3": "0.9.11-grist.1",
"@gristlabs/express-session": "1.17.0",
"@gristlabs/pidusage": "2.0.17",
@ -84,6 +86,7 @@
"diff-match-patch": "1.0.5",
"double-ended-queue": "2.1.0-0",
"electron": "3.0.7",
"exceljs": "4.2.1",
"express": "4.16.4",
"file-type": "14.1.4",
"fs-extra": "7.0.0",

@ -0,0 +1,15 @@
<!doctype html>
<html>
<head>
<meta charset="utf8">
</head>
<body>
<!-- INSERT CONFIG -->
<!-- INSERT MESSAGE -->
<script>
// Determine proper home url for origin parameter
const origin = gristConfig.pathOnly ? gristConfig.homeUrl : `https://${gristConfig.org}${gristConfig.baseDomain}`;
window.opener.postMessage(message, origin);
</script>
</body>
</html>
Loading…
Cancel
Save