mirror of
https://github.com/gristlabs/grist-core.git
synced 2026-03-02 04:09:24 +00:00
(core) Download as CSV button on sections
Summary: Adding "Download as CSV" button that exports filtred section data to csv Test Plan: Browser tests Reviewers: paulfitz, dsagal Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D2830
This commit is contained in:
Jarosław Sadziński
3
app/server/declarations.d.ts
vendored
3
app/server/declarations.d.ts
vendored
@@ -1,7 +1,6 @@
|
||||
declare module "app/server/lib/ActionLog";
|
||||
declare module "app/server/lib/sandboxUtil";
|
||||
declare module "app/server/lib/User";
|
||||
declare module "app/server/serverMethods";
|
||||
|
||||
declare module "app/server/lib/Comm" {
|
||||
import {Client, ClientMethod} from "app/server/lib/Client";
|
||||
@@ -99,3 +98,5 @@ declare module '@gristlabs/pidusage' {
|
||||
import * as pidusage from 'pidusage';
|
||||
export = pidusage;
|
||||
}
|
||||
|
||||
declare module "csv";
|
||||
|
||||
@@ -30,8 +30,8 @@ export class DocWorker {
|
||||
this._comm = comm;
|
||||
}
|
||||
|
||||
public getCSV(req: express.Request, res: express.Response): void {
|
||||
return generateCSV(req, res, this._comm);
|
||||
public async getCSV(req: express.Request, res: express.Response): Promise<void> {
|
||||
await generateCSV(req, res, this._comm);
|
||||
}
|
||||
|
||||
public async getAttachment(req: express.Request, res: express.Response): Promise<void> {
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
const gutil = require('app/common/gutil');
|
||||
const {SortFunc} = require('app/common/SortFunc');
|
||||
const ValueFormatter = require('app/common/ValueFormatter');
|
||||
const {docSessionFromRequest} = require('app/server/lib/DocSession');
|
||||
const Promise = require('bluebird');
|
||||
const contentDisposition = require('content-disposition');
|
||||
const csv = require('csv');
|
||||
const fs = require('fs-extra');
|
||||
const log = require('./lib/log');
|
||||
const {ServerColumnGetters} = require('./lib/ServerColumnGetters');
|
||||
const multiparty = require('multiparty');
|
||||
const tmp = require('tmp');
|
||||
const _ = require('underscore');
|
||||
|
||||
Promise.promisifyAll(csv);
|
||||
Promise.promisifyAll(multiparty, {filter: name => (name === 'parse'), multiArgs: true});
|
||||
Promise.promisifyAll(fs);
|
||||
Promise.promisifyAll(tmp);
|
||||
|
||||
function generateCSV(req, res, comm) {
|
||||
log.info('Generating .csv file...');
|
||||
// Get the current table id
|
||||
var tableId = req.param('tableId');
|
||||
var viewSectionId = parseInt(req.param('viewSection'), 10);
|
||||
var activeSortOrder = gutil.safeJsonParse(req.param('activeSortSpec'), null);
|
||||
|
||||
// Get the active doc
|
||||
var clientId = req.param('clientId');
|
||||
var docFD = parseInt(req.param('docFD'), 10);
|
||||
var client = comm.getClient(clientId);
|
||||
var docSession = client.getDocSession(docFD);
|
||||
var activeDoc = docSession.activeDoc;
|
||||
|
||||
// Generate a decent name for the exported file.
|
||||
var docName = req.query.title || activeDoc.docName;
|
||||
var name = docName +
|
||||
(tableId === docName ? '' : '-' + tableId) + '.csv';
|
||||
|
||||
res.set('Content-Type', 'text/csv');
|
||||
res.setHeader('Content-Disposition', contentDisposition(name));
|
||||
return makeCSV(activeDoc, viewSectionId, activeSortOrder, req)
|
||||
.then(data => res.send(data));
|
||||
}
|
||||
exports.generateCSV = generateCSV;
|
||||
|
||||
/**
|
||||
* Returns a csv stream that can be transformed or parsed. See https://github.com/wdavidw/node-csv
|
||||
* for API details.
|
||||
*
|
||||
* @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
|
||||
* @param {Integer} viewSectionId - id of the viewsection to export.
|
||||
* @param {Integer[]} activeSortOrder (optional) - overriding sort order.
|
||||
* @return {Promise<string>} Promise for the resulting CSV.
|
||||
*/
|
||||
function makeCSV(activeDoc, viewSectionId, sortOrder, req) {
|
||||
return Promise.try(() => {
|
||||
const tables = activeDoc.docData.getTables();
|
||||
const viewSection = tables.get('_grist_Views_section').getRecord(viewSectionId);
|
||||
const table = tables.get('_grist_Tables').getRecord(viewSection.tableRef);
|
||||
const fields = tables.get('_grist_Views_section_field').filterRecords({ parentId: viewSection.id});
|
||||
const tableColumns = tables.get('_grist_Tables_column').filterRecords({parentId: table.id});
|
||||
const tableColsById = _.indexBy(tableColumns, 'id');
|
||||
|
||||
// Produce a column description matching what user will see / expect to export
|
||||
const viewify = (col, field) => {
|
||||
field = field || {};
|
||||
const displayCol = tableColsById[field.displayCol || col.displayCol || col.id];
|
||||
const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {});
|
||||
const fieldWidgetOptions = gutil.safeJsonParse(field.widgetOptions, {});
|
||||
return {
|
||||
id: displayCol.id,
|
||||
colId: displayCol.colId,
|
||||
label: col.label,
|
||||
colType: col.type,
|
||||
widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions)
|
||||
};
|
||||
};
|
||||
const viewColumns = _.sortBy(fields, 'parentPos').map(
|
||||
(field) => viewify(tableColsById[field.colRef], field));
|
||||
|
||||
// The columns named in sort order need to now become display columns
|
||||
sortOrder = sortOrder || gutil.safeJsonParse(viewSection.sortColRefs, []);
|
||||
const fieldsByColRef = _.indexBy(fields, 'colRef');
|
||||
sortOrder = sortOrder.map((directionalColRef) => {
|
||||
const colRef = Math.abs(directionalColRef);
|
||||
const col = tableColsById[colRef];
|
||||
if (!col) return 0;
|
||||
const effectiveColRef = viewify(col, fieldsByColRef[colRef]).id;
|
||||
return directionalColRef > 0 ? effectiveColRef : -effectiveColRef;
|
||||
});
|
||||
|
||||
return [activeDoc.fetchTable(docSessionFromRequest(req), table.tableId, true), tableColumns, viewColumns];
|
||||
}).spread((data, tableColumns, viewColumns) => {
|
||||
const rowIds = data[2];
|
||||
const dataByColId = data[3];
|
||||
const getters = new ServerColumnGetters(rowIds, dataByColId, tableColumns);
|
||||
const sorter = new SortFunc(getters);
|
||||
sorter.updateSpec(sortOrder);
|
||||
rowIds.sort((a, b) => sorter.compare(a, b));
|
||||
const formatters = viewColumns.map(col =>
|
||||
ValueFormatter.createFormatter(col.colType, col.widgetOptions));
|
||||
// Arrange the data into a row-indexed matrix, starting with column headers.
|
||||
const csvMatrix = [viewColumns.map(col => col.label)];
|
||||
const access = viewColumns.map(col => getters.getColGetter(col.id));
|
||||
rowIds.forEach(row => {
|
||||
csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter(row))));
|
||||
});
|
||||
return csv.stringifyAsync(csvMatrix);
|
||||
});
|
||||
}
|
||||
exports.makeCSV = makeCSV;
|
||||
209
app/server/serverMethods.ts
Normal file
209
app/server/serverMethods.ts
Normal file
@@ -0,0 +1,209 @@
|
||||
import * as gutil from 'app/common/gutil';
|
||||
import { SortFunc } from "app/common/SortFunc";
|
||||
import { docSessionFromRequest } from "app/server/lib/DocSession";
|
||||
import * as bluebird from "bluebird";
|
||||
import * as contentDisposition from "content-disposition";
|
||||
import * as csv from "csv";
|
||||
import * as log from "./lib/log";
|
||||
import { ServerColumnGetters } from "./lib/ServerColumnGetters";
|
||||
import * as _ from "underscore";
|
||||
import * as express from "express";
|
||||
import * as Comm from 'app/server/lib/Comm';
|
||||
import { ActiveDoc } from "app/server/lib/ActiveDoc";
|
||||
import { createFormatter } from "app/common/ValueFormatter";
|
||||
import { SchemaTypes } from "app/common/schema";
|
||||
import { RequestWithLogin } from "app/server/lib/Authorizer";
|
||||
import { RowRecord } from "app/common/DocActions";
|
||||
import { buildColFilter } from "app/common/ColumnFilterFunc";
|
||||
import { buildRowFilter } from "app/common/RowFilterFunc";
|
||||
|
||||
// promisify csv
|
||||
bluebird.promisifyAll(csv);
|
||||
|
||||
export async function generateCSV(req: express.Request, res: express.Response, comm: Comm) {
|
||||
|
||||
log.info('Generating .csv file...');
|
||||
// Get the current table id
|
||||
const tableId = req.param('tableId');
|
||||
const viewSectionId = parseInt(req.param('viewSection'), 10);
|
||||
const activeSortOrder = gutil.safeJsonParse(req.param('activeSortSpec'), null);
|
||||
const filters: Filter[] = gutil.safeJsonParse(req.param("filters"), []) || [];
|
||||
|
||||
// Get the active doc
|
||||
const clientId = req.param('clientId');
|
||||
const docFD = parseInt(req.param('docFD'), 10);
|
||||
const client = comm.getClient(clientId);
|
||||
const docSession = client.getDocSession(docFD);
|
||||
const activeDoc = docSession.activeDoc;
|
||||
|
||||
// Generate a decent name for the exported file.
|
||||
const docName = req.query.title || activeDoc.docName;
|
||||
const name = docName +
|
||||
(tableId === docName ? '' : '-' + tableId) + '.csv';
|
||||
|
||||
try {
|
||||
const data = await makeCSV(activeDoc, viewSectionId, activeSortOrder, filters, req);
|
||||
res.set('Content-Type', 'text/csv');
|
||||
res.setHeader('Content-Disposition', contentDisposition(name));
|
||||
res.send(data);
|
||||
} catch (err) {
|
||||
log.error("Exporting to CSV has failed. Request url: %s", req.url, err);
|
||||
// send a generic information to client
|
||||
const errHtml =
|
||||
`<!doctype html>
|
||||
<html>
|
||||
<body>There was an unexpected error while generating a csv file.</body>
|
||||
</html>
|
||||
`;
|
||||
res.status(400).send(errHtml);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a csv stream that can be transformed or parsed. See https://github.com/wdavidw/node-csv
|
||||
* for API details.
|
||||
*
|
||||
* @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
|
||||
* @param {Integer} viewSectionId - id of the viewsection to export.
|
||||
* @param {Integer[]} activeSortOrder (optional) - overriding sort order.
|
||||
* @return {Promise<string>} Promise for the resulting CSV.
|
||||
*/
|
||||
export async function makeCSV(
|
||||
activeDoc: ActiveDoc,
|
||||
viewSectionId: number,
|
||||
sortOrder: number[],
|
||||
filters: Filter[],
|
||||
req: express.Request) {
|
||||
|
||||
const {
|
||||
table,
|
||||
viewSection,
|
||||
tableColumns,
|
||||
fields
|
||||
} = explodeSafe(activeDoc, viewSectionId);
|
||||
|
||||
const tableColsById = _.indexBy(tableColumns, 'id');
|
||||
|
||||
// Produce a column description matching what user will see / expect to export
|
||||
const viewify = (col: GristTablesColumn, field: GristViewsSectionField) => {
|
||||
field = field || {};
|
||||
const displayCol = tableColsById[field.displayCol || col.displayCol || col.id];
|
||||
const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {});
|
||||
const fieldWidgetOptions = gutil.safeJsonParse(field.widgetOptions, {});
|
||||
const filterFunc = buildColFilter(filters.find(x => x.colRef === field.colRef)?.filter);
|
||||
return {
|
||||
id: displayCol.id,
|
||||
colId: displayCol.colId,
|
||||
label: col.label,
|
||||
colType: col.type,
|
||||
filterFunc,
|
||||
widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions)
|
||||
};
|
||||
};
|
||||
const viewColumns = _.sortBy(fields, 'parentPos').map(
|
||||
(field) => viewify(tableColsById[field.colRef], field));
|
||||
|
||||
// The columns named in sort order need to now become display columns
|
||||
sortOrder = sortOrder || gutil.safeJsonParse(viewSection.sortColRefs, []);
|
||||
const fieldsByColRef = _.indexBy(fields, 'colRef');
|
||||
sortOrder = sortOrder.map((directionalColRef) => {
|
||||
const colRef = Math.abs(directionalColRef);
|
||||
const col = tableColsById[colRef];
|
||||
if (!col) {
|
||||
return 0;
|
||||
}
|
||||
const effectiveColRef = viewify(col, fieldsByColRef[colRef]).id;
|
||||
return directionalColRef > 0 ? effectiveColRef : -effectiveColRef;
|
||||
});
|
||||
|
||||
const data = await activeDoc.fetchTable(docSessionFromRequest(req as RequestWithLogin), table.tableId, true);
|
||||
const rowIds = data[2];
|
||||
const dataByColId = data[3];
|
||||
const getters = new ServerColumnGetters(rowIds, dataByColId, tableColumns);
|
||||
const sorter = new SortFunc(getters);
|
||||
sorter.updateSpec(sortOrder);
|
||||
rowIds.sort((a, b) => sorter.compare(a, b));
|
||||
const formatters = viewColumns.map(col =>
|
||||
createFormatter(col.colType, col.widgetOptions));
|
||||
// Arrange the data into a row-indexed matrix, starting with column headers.
|
||||
const csvMatrix = [viewColumns.map(col => col.label)];
|
||||
const access = viewColumns.map(col => getters.getColGetter(col.id));
|
||||
// create row filter based on all columns filter
|
||||
const rowFilter = viewColumns
|
||||
.map((col, c) => buildRowFilter(access[c], col.filterFunc))
|
||||
.reduce((prevFilter, curFilter) => (id) => prevFilter(id) && curFilter(id), () => true);
|
||||
rowIds.forEach(row => {
|
||||
if (!rowFilter(row)) {
|
||||
return;
|
||||
}
|
||||
csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter!(row))));
|
||||
});
|
||||
|
||||
return csv.stringifyAsync(csvMatrix);
|
||||
}
|
||||
|
||||
|
||||
// helper method that retrieves various parts about view section
|
||||
// from ActiveDoc
|
||||
function explodeSafe(activeDoc: ActiveDoc, viewSectionId: number) {
|
||||
const docData = activeDoc.docData;
|
||||
|
||||
if (!docData) {
|
||||
// Should not happen unless there's a logic error
|
||||
// This method is exported (for testing) so it is possible
|
||||
// to call it without loading active doc first
|
||||
throw new Error("Document hasn't been loaded yet");
|
||||
}
|
||||
|
||||
const viewSection = docData
|
||||
.getTable('_grist_Views_section')
|
||||
?.getRecord(viewSectionId) as GristViewsSection | undefined;
|
||||
|
||||
if (!viewSection) {
|
||||
throw new Error(`No table '_grist_Views_section' in document with id ${activeDoc.docName}`);
|
||||
}
|
||||
|
||||
const table = docData
|
||||
.getTable('_grist_Tables')
|
||||
?.getRecord(viewSection.tableRef) as GristTables | undefined;
|
||||
|
||||
if (!table) {
|
||||
throw new Error(`No table '_grist_Tables' in document with id ${activeDoc.docName}`);
|
||||
}
|
||||
|
||||
const fields = docData
|
||||
.getTable('_grist_Views_section_field')
|
||||
?.filterRecords({ parentId: viewSection.id }) as GristViewsSectionField[] | undefined;
|
||||
|
||||
if (!fields) {
|
||||
throw new Error(`No table '_grist_Views_section_field' in document with id ${activeDoc.docName}`);
|
||||
}
|
||||
|
||||
const tableColumns = docData
|
||||
.getTable('_grist_Tables_column')
|
||||
?.filterRecords({ parentId: table.id }) as GristTablesColumn[] | undefined;
|
||||
|
||||
if (!tableColumns) {
|
||||
throw new Error(`No table '_grist_Tables_column' in document with id ${activeDoc.docName}`);
|
||||
}
|
||||
|
||||
return {
|
||||
table,
|
||||
fields,
|
||||
tableColumns,
|
||||
viewSection
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// Type helpers for types used in this export
|
||||
type RowModel<TName extends keyof SchemaTypes> = RowRecord & {
|
||||
[ColId in keyof SchemaTypes[TName]]: SchemaTypes[TName][ColId];
|
||||
};
|
||||
type GristViewsSection = RowModel<'_grist_Views_section'>
|
||||
type GristTables = RowModel<'_grist_Tables'>
|
||||
type GristViewsSectionField = RowModel<'_grist_Views_section_field'>
|
||||
type GristTablesColumn = RowModel<'_grist_Tables_column'>
|
||||
|
||||
// Type for filters passed from the client
|
||||
interface Filter { colRef: number, filter: string }
|
||||
Reference in New Issue
Block a user