Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

Option to export colId as header in CSV / XSLX instead of label (#688) (#692)

Browse Source
This commit is contained in:
Florent 2023-10-16 02:17:43 +02:00 committed by GitHub
parent 90ce8825d9
commit eb55afcbc4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 126 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/server/lib/DocApi.ts
View File

@ -1202,12 +1202,13 @@ export class DocWorkerApi {
this._app.get('/api/docs/:docId/download/xlsx', canView, withDoc(async (activeDoc, req, res) => {
// Query DB for doc metadata to get the doc title (to use as the filename).
const {name: docTitle} = await this._dbManager.getDoc(req);
const options = !_.isEmpty(req.query) ? this._getDownloadOptions(req, docTitle) : {
const options: DownloadOptions = !_.isEmpty(req.query) ? this._getDownloadOptions(req, docTitle) : {
filename: docTitle,
tableId: '',
viewSectionId: undefined,
filters: [],
sortOrder: [],
header: 'label'
};
await downloadXLSX(activeDoc, req, res, options);
}));

7
app/server/lib/Export.ts
View File

@ -17,7 +17,7 @@ import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFor
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {RequestWithLogin} from 'app/server/lib/Authorizer';
import {docSessionFromRequest} from 'app/server/lib/DocSession';
import {optIntegerParam, optJsonParam, stringParam} from 'app/server/lib/requestUtils';
import {optIntegerParam, optJsonParam, optStringParam, stringParam} from 'app/server/lib/requestUtils';
import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters';
import * as express from 'express';
import * as _ from 'underscore';
@ -90,6 +90,8 @@ export interface ExportData {
docSettings: DocumentSettings;
}
export type ExportHeader = 'colId' | 'label';
/**
* Export parameters that identifies a section, filters, sort order.
*/
@ -99,6 +101,7 @@ export interface ExportParameters {
sortOrder?: number[];
filters?: Filter[];
linkingFilter?: FilterColValues;
header?: ExportHeader;
}
/**
@ -117,6 +120,7 @@ export function parseExportParameters(req: express.Request): ExportParameters {
const sortOrder = optJsonParam(req.query.activeSortSpec, []) as number[];
const filters: Filter[] = optJsonParam(req.query.filters, []);
const linkingFilter: FilterColValues = optJsonParam(req.query.linkingFilter, null);
const header = optStringParam(req.query.header, 'header', {allowed: ['label', 'colId']}) as ExportHeader | undefined;
return {
tableId,
@ -124,6 +128,7 @@ export function parseExportParameters(req: express.Request): ExportParameters {
sortOrder,
filters,
linkingFilter,
header,
};
}

57
app/server/lib/ExportCSV.ts
View File

@ -1,7 +1,7 @@
import {ApiError} from 'app/common/ApiError';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {FilterColValues} from "app/common/ActiveDocAPI";
import {DownloadOptions, ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export';
import {DownloadOptions, ExportData, ExportHeader, exportSection, exportTable, Filter} from 'app/server/lib/Export';
import log from 'app/server/lib/log';
import * as bluebird from 'bluebird';
import contentDisposition from 'content-disposition';
@ -17,11 +17,13 @@ bluebird.promisifyAll(csv);
export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
res: express.Response, options: DownloadOptions) {
log.info('Generating .csv file...');
const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, header} = options;
const data = viewSectionId ?
await makeCSVFromViewSection(
activeDoc, viewSectionId, sortOrder || null, filters || null, linkingFilter || null, req) :
await makeCSVFromTable(activeDoc, tableId, req);
await makeCSVFromViewSection({
activeDoc, viewSectionId, sortOrder: sortOrder || null, filters: filters || null,
linkingFilter: linkingFilter || null, header, req
}) :
await makeCSVFromTable({activeDoc, tableId, header, req});
res.set('Content-Type', 'text/csv');
res.setHeader('Content-Disposition', contentDisposition(filename + '.csv'));
res.send(data);
@ -32,36 +34,51 @@ export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
*
* See https://github.com/wdavidw/node-csv for API details.
*
* @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
* @param {Integer} viewSectionId - id of the viewsection to export.
* @param {Integer[]} activeSortOrder (optional) - overriding sort order.
* @param {Filter[]} filters (optional) - filters defined from ui.
* @param {Object} options - options for the export.
* @param {Object} options.activeDoc - the activeDoc that the table being converted belongs to.
* @param {Integer} options.viewSectionId - id of the viewsection to export.
* @param {Integer[]} options.activeSortOrder (optional) - overriding sort order.
* @param {Filter[]} options.filters (optional) - filters defined from ui.
* @param {FilterColValues} options.linkingFilter (optional) - linking filter defined from ui.
* @param {string} options.header (optional) - which field of the column to use as header
* @param {express.Request} options.req - the request object.
*
* @return {Promise<string>} Promise for the resulting CSV.
*/
export async function makeCSVFromViewSection(
export async function makeCSVFromViewSection({
activeDoc, viewSectionId, sortOrder = null, filters = null, linkingFilter = null, header, req
}: {
activeDoc: ActiveDoc,
viewSectionId: number,
sortOrder: number[] | null,
filters: Filter[] | null,
linkingFilter: FilterColValues | null,
req: express.Request) {
header?: ExportHeader,
req: express.Request
}) {
const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, linkingFilter, req);
const file = convertToCsv(data);
const file = convertToCsv(data, { header });
return file;
}
/**
* Returns a csv stream of a table that can be transformed or parsed.
*
* @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
* @param {Integer} tableId - id of the table to export.
* @param {Object} options - options for the export.
* @param {Object} options.activeDoc - the activeDoc that the table being converted belongs to.
* @param {Integer} options.tableId - id of the table to export.
* @param {string} options.header (optional) - which field of the column to use as header
* @param {express.Request} options.req - the request object.
*
* @return {Promise<string>} Promise for the resulting CSV.
*/
export async function makeCSVFromTable(
export async function makeCSVFromTable({ activeDoc, tableId, header, req }: {
activeDoc: ActiveDoc,
tableId: string,
req: express.Request) {
header?: ExportHeader,
req: express.Request
}) {
if (!activeDoc.docData) {
throw new Error('No docData in active document');
@ -76,7 +93,7 @@ export async function makeCSVFromTable(
}
const data = await exportTable(activeDoc, tableRef, req);
const file = convertToCsv(data);
const file = convertToCsv(data, { header });
return file;
}
@ -84,13 +101,13 @@ function convertToCsv({
rowIds,
access,
columns: viewColumns,
docSettings
}: ExportData) {
}: ExportData, options: { header?: ExportHeader }) {
// create formatters for columns
const formatters = viewColumns.map(col => col.formatter);
// Arrange the data into a row-indexed matrix, starting with column headers.
const csvMatrix = [viewColumns.map(col => col.label)];
const colPropertyAsHeader = options.header ?? 'label';
const csvMatrix = [viewColumns.map(col => col[colPropertyAsHeader])];
// populate all the rows with values as strings
rowIds.forEach(row => {
csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter(row))));

64
app/server/lib/workerExporter.ts
View File

@ -1,7 +1,7 @@
import {PassThrough} from 'stream';
import {FilterColValues} from "app/common/ActiveDocAPI";
import {ActiveDocSource, doExportDoc, doExportSection, doExportTable,
ExportData, ExportParameters, Filter} from 'app/server/lib/Export';
ExportData, ExportHeader, ExportParameters, Filter} from 'app/server/lib/Export';
import {createExcelFormatter} from 'app/server/lib/ExcelFormatter';
import * as log from 'app/server/lib/log';
import {Alignment, Border, Buffer as ExcelBuffer, stream as ExcelWriteStream,
@ -79,26 +79,34 @@ export async function doMakeXLSXFromOptions(
stream: Stream,
options: ExportParameters
) {
const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options;
const {tableId, viewSectionId, filters, sortOrder, linkingFilter, header} = options;
if (viewSectionId) {
return doMakeXLSXFromViewSection(activeDocSource, testDates, stream, viewSectionId,
sortOrder || null, filters || null, linkingFilter || null);
return doMakeXLSXFromViewSection({activeDocSource, testDates, stream, viewSectionId, header,
sortOrder: sortOrder || null, filters: filters || null, linkingFilter: linkingFilter || null});
} else if (tableId) {
return doMakeXLSXFromTable(activeDocSource, testDates, stream, tableId);
return doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, header});
} else {
return doMakeXLSX(activeDocSource, testDates, stream);
return doMakeXLSX({activeDocSource, testDates, stream, header});
}
}
/**
* @async
* Returns a XLSX stream of a view section that can be transformed or parsed.
*
* @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
* @param {Integer} viewSectionId - id of the viewsection to export.
* @param {Integer[]} activeSortOrder (optional) - overriding sort order.
* @param {Filter[]} filters (optional) - filters defined from ui.
* @param {Object} options - options for the export.
* @param {Object} options.activeDocSource - the activeDoc that the table being converted belongs to.
* @param {Integer} options.viewSectionId - id of the viewsection to export.
* @param {Integer[]} options.activeSortOrder (optional) - overriding sort order.
* @param {Filter[]} options.filters (optional) - filters defined from ui.
* @param {FilterColValues} options.linkingFilter (optional)
* @param {Stream} options.stream - the stream to write to.
* @param {boolean} options.testDates - whether to use static dates for testing.
* @param {string} options.header (optional) - which field of the column to use as header
*/
async function doMakeXLSXFromViewSection(
async function doMakeXLSXFromViewSection({
activeDocSource, testDates, stream, viewSectionId, sortOrder, filters, linkingFilter, header
}: {
activeDocSource: ActiveDocSource,
testDates: boolean,
stream: Stream,
@ -106,27 +114,35 @@ async function doMakeXLSXFromViewSection(
sortOrder: number[] | null,
filters: Filter[] | null,
linkingFilter: FilterColValues | null,
) {
header?: ExportHeader,
}) {
const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters, linkingFilter);
const {exportTable, end} = convertToExcel(stream, testDates);
const {exportTable, end} = convertToExcel(stream, testDates, {header});
exportTable(data);
return end();
}
/**
* @async
* Returns a XLSX stream of a table that can be transformed or parsed.
*
* @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
* @param {Integer} tableId - id of the table to export.
* @param {Object} options - options for the export.
* @param {Object} options.activeDocSource - the activeDoc that the table being converted belongs to.
* @param {Integer} options.tableId - id of the table to export.
* @param {Stream} options.stream - the stream to write to.
* @param {boolean} options.testDates - whether to use static dates for testing.
* @param {string} options.header (optional) - which field of the column to use as header
*
*/
async function doMakeXLSXFromTable(
async function doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, header}: {
activeDocSource: ActiveDocSource,
testDates: boolean,
stream: Stream,
tableId: string,
) {
header?: ExportHeader,
}) {
const data = await doExportTable(activeDocSource, {tableId});
const {exportTable, end} = convertToExcel(stream, testDates);
const {exportTable, end} = convertToExcel(stream, testDates, {header});
exportTable(data);
return end();
}
@ -134,12 +150,13 @@ async function doMakeXLSXFromTable(
/**
* Creates excel document with all tables from an active Grist document.
*/
async function doMakeXLSX(
async function doMakeXLSX({activeDocSource, testDates, stream, header}: {
activeDocSource: ActiveDocSource,
testDates: boolean,
stream: Stream,
): Promise<void|ExcelBuffer> {
const {exportTable, end} = convertToExcel(stream, testDates);
header?: ExportHeader,
}): Promise<void|ExcelBuffer> {
const {exportTable, end} = convertToExcel(stream, testDates, {header});
await doExportDoc(activeDocSource, async (table: ExportData) => exportTable(table));
return end();
}
@ -152,7 +169,7 @@ async function doMakeXLSX(
* (The second option is for grist-static; at the time of writing
* WorkbookWriter doesn't appear to be available in a browser context).
*/
function convertToExcel(stream: Stream|undefined, testDates: boolean): {
function convertToExcel(stream: Stream|undefined, testDates: boolean, options: { header?: ExportHeader }): {
exportTable: (table: ExportData) => void,
end: () => Promise<void|ExcelBuffer>,
} {
@ -206,7 +223,8 @@ function convertToExcel(stream: Stream|undefined, testDates: boolean): {
const formatters = columns.map(col => createExcelFormatter(col.formatter.type, col.formatter.widgetOpts));
// Generate headers for all columns with correct styles for whole column.
// Actual header style for a first row will be overwritten later.
ws.columns = columns.map((col, c) => ({ header: col.label, style: formatters[c].style() }));
const colHeader = options.header ?? 'label';
ws.columns = columns.map((col, c) => ({ header: col[colHeader], style: formatters[c].style() }));
// style up the header row
for (let i = 1; i <= columns.length; i++) {
// apply to all rows (including header)

53
test/server/lib/DocApi.ts
View File

@ -230,6 +230,14 @@ describe('DocApi', function () {
// Contains the tests. This is where you want to add more test.
function testDocApi() {
async function generateDocAndUrl(docName: string = "Dummy") {
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({name: docName}, wid);
const docUrl = `${serverUrl}/api/docs/${docId}`;
const tableUrl = `${serverUrl}/api/docs/${docId}/tables/Table1`;
return { docUrl, tableUrl, docId };
}
it("creator should be owner of a created ws", async () => {
const kiwiEmail = 'kiwi@getgrist.com';
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
@ -1080,13 +1088,13 @@ function testDocApi() {
});
describe("/docs/{did}/tables/{tid}/columns", function () {
async function generateDocAndUrl(docName: string = "Dummy") {
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({name: docName}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/columns`;
return { url, docId };
async function generateDocAndUrlForColumns(name: string) {
const { tableUrl, docId } = await generateDocAndUrl(name);
return {
docId,
url: `${tableUrl}/columns`,
};
}
describe("PUT /docs/{did}/tables/{tid}/columns", function () {
async function getColumnFieldsMapById(url: string, params: any) {
const result = await axios.get(url, {...chimpy, params});
@ -1104,7 +1112,7 @@ function testDocApi() {
expectedFieldsByColId: Record<string, object>,
opts?: { getParams?: any }
) {
const {url} = await generateDocAndUrl('ColumnsPut');
const {url} = await generateDocAndUrlForColumns('ColumnsPut');
const body: ColumnsPut = { columns };
const resp = await axios.put(url, body, {...chimpy, params});
assert.equal(resp.status, 200);
@ -1175,7 +1183,7 @@ function testDocApi() {
it('should forbid update by viewers', async function () {
// given
const { url, docId } = await generateDocAndUrl('ColumnsPut');
const { url, docId } = await generateDocAndUrlForColumns('ColumnsPut');
await userApi.updateDocPermissions(docId, {users: {'kiwi@getgrist.com': 'viewers'}});
// when
@ -1187,7 +1195,7 @@ function testDocApi() {
it("should return 404 when table is not found", async function() {
// given
const { url } = await generateDocAndUrl('ColumnsPut');
const { url } = await generateDocAndUrlForColumns('ColumnsPut');
const notFoundUrl = url.replace("Table1", "NonExistingTable");
// when
@ -1201,7 +1209,7 @@ function testDocApi() {
describe("DELETE /docs/{did}/tables/{tid}/columns/{colId}", function () {
it('should delete some column', async function() {
const {url} = await generateDocAndUrl('ColumnDelete');
const {url} = await generateDocAndUrlForColumns('ColumnDelete');
const deleteUrl = url + '/A';
const resp = await axios.delete(deleteUrl, chimpy);
@ -1215,7 +1223,7 @@ function testDocApi() {
});
it('should return 404 if table not found', async function() {
const {url} = await generateDocAndUrl('ColumnDelete');
const {url} = await generateDocAndUrlForColumns('ColumnDelete');
const deleteUrl = url.replace("Table1", "NonExistingTable") + '/A';
const resp = await axios.delete(deleteUrl, chimpy);
@ -1224,7 +1232,7 @@ function testDocApi() {
});
it('should return 404 if column not found', async function() {
const {url} = await generateDocAndUrl('ColumnDelete');
const {url} = await generateDocAndUrlForColumns('ColumnDelete');
const deleteUrl = url + '/NonExistingColId';
const resp = await axios.delete(deleteUrl, chimpy);
@ -1233,7 +1241,7 @@ function testDocApi() {
});
it('should forbid column deletion by viewers', async function() {
const {url, docId} = await generateDocAndUrl('ColumnDelete');
const {url, docId} = await generateDocAndUrlForColumns('ColumnDelete');
await userApi.updateDocPermissions(docId, {users: {'kiwi@getgrist.com': 'viewers'}});
const deleteUrl = url + '/A';
const resp = await axios.delete(deleteUrl, kiwi);
@ -2609,6 +2617,25 @@ function testDocApi() {
assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n');
});
it('GET /docs/{did}/download/csv with header=colId shows columns id in the header instead of their name',
async function () {
const { docUrl } = await generateDocAndUrl('csvWithColIdAsHeader');
const AColRef = 2;
const userActions = [
['AddRecord', 'Table1', null, {A: 'a1', B: 'b1'}],
['UpdateRecord', '_grist_Tables_column', AColRef, { untieColIdFromLabel: true }],
['UpdateRecord', '_grist_Tables_column', AColRef, {
label: 'Column label for A',
colId: 'AColId'
}]
];
const resp = await axios.post(`${docUrl}/apply`, userActions, chimpy);
assert.equal(resp.status, 200);
const csvResp = await axios.get(`${docUrl}/download/csv?tableId=Table1&header=colId`, chimpy);
assert.equal(csvResp.status, 200);
assert.equal(csvResp.data, 'AColId,B,C\na1,b1,\n');
});
it("GET /docs/{did}/download/csv respects permissions", async function () {
// kiwi has no access to TestDoc
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1`, kiwi);