diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index fb86857c..13549f07 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -576,13 +576,19 @@ export class GristDoc extends DisposableWithEvents { } public getCsvLink() { - return this.docComm.docUrl('gen_csv') + '?' + encodeQueryParams({ + const filters = this.viewModel.activeSection.peek().filteredFields.get().map(field=> ({ + colRef : field.colRef.peek(), + filter : field.activeFilter.peek() + })); + const params = { ...this.docComm.getUrlParams(), title: this.docPageModel.currentDocTitle.get(), viewSection: this.viewModel.activeSectionId(), tableId: this.viewModel.activeSection().table().tableId(), - activeSortSpec: JSON.stringify(this.viewModel.activeSection().activeSortSpec()) - }); + activeSortSpec: JSON.stringify(this.viewModel.activeSection().activeSortSpec()), + filters : JSON.stringify(filters), + }; + return this.docComm.docUrl('gen_csv') + '?' + encodeQueryParams(params); } public hasGranularAccessRules(): boolean { diff --git a/app/client/models/ColumnFilter.ts b/app/client/models/ColumnFilter.ts index 7ca174d0..eb7ad413 100644 --- a/app/client/models/ColumnFilter.ts +++ b/app/client/models/ColumnFilter.ts @@ -1,53 +1,8 @@ import {CellValue} from 'app/common/DocActions'; import {nativeCompare} from 'app/common/gutil'; import {Computed, Disposable, Observable} from 'grainjs'; - -export type ColumnFilterFunc = (value: CellValue) => boolean; - -interface FilterSpec { // Filter object as stored in the db - included?: CellValue[]; - excluded?: CellValue[]; -} - -// A more efficient representation of filter state for a column than FilterSpec. -interface FilterState { - include: boolean; - values: Set; -} - -// Creates a FilterState. Accepts spec as a json string or a FilterSpec. -function makeFilterState(spec: string | FilterSpec): FilterState { - if (typeof(spec) === 'string') { - return makeFilterState((spec && JSON.parse(spec)) || {}); - } - return { - include: Boolean(spec.included), - values: new Set(spec.included || spec.excluded || []), - }; -} - -// Returns true if state and spec are equivalent, false otherwise. -export function isEquivalentFilter(state: FilterState, spec: FilterSpec): boolean { - const other = makeFilterState(spec); - if (state.include !== other.include) { return false; } - if (state.values.size !== other.values.size) { return false; } - for (const val of other.values) { if (!state.values.has(val)) { return false; } } - return true; -} - -// Returns a filter function for a particular column: the function takes a cell value and returns -// whether it's accepted according to the given FilterState. -function makeFilterFunc({include, values}: FilterState): ColumnFilterFunc { - // NOTE: This logic results in complex values and their stringified JSON representations as equivalent. - // For example, a TypeError in the formula column and the string '["E","TypeError"]' would be seen as the same. - // TODO: This narrow corner case seems acceptable for now, but may be worth revisiting. - return (val: CellValue) => (values.has(Array.isArray(val) ? JSON.stringify(val) : val) === include); -} - -// Given a JSON string, returns a ColumnFilterFunc -export function getFilterFunc(filterJson: string): ColumnFilterFunc|null { - return filterJson ? makeFilterFunc(makeFilterState(filterJson)) : null; -} +import {ColumnFilterFunc, makeFilterFunc} from "app/common/ColumnFilterFunc"; +import {FilterSpec, FilterState, makeFilterState} from "app/common/FilterState"; /** * ColumnFilter implements a custom filter on a column, i.e. a filter that's diverged from what's diff --git a/app/client/models/QuerySet.ts b/app/client/models/QuerySet.ts index 7a3777d2..96a91233 100644 --- a/app/client/models/QuerySet.ts +++ b/app/client/models/QuerySet.ts @@ -28,7 +28,7 @@ */ import * as DataTableModel from 'app/client/models/DataTableModel'; import {DocModel} from 'app/client/models/DocModel'; -import {BaseFilteredRowSource, FilterFunc, RowId, RowList, RowSource} from 'app/client/models/rowset'; +import {BaseFilteredRowSource, RowId, RowList, RowSource} from 'app/client/models/rowset'; import {TableData} from 'app/client/models/TableData'; import {ActiveDocAPI, Query} from 'app/common/ActiveDocAPI'; import {TableDataAction} from 'app/common/DocActions'; @@ -36,6 +36,7 @@ import {DocData} from 'app/common/DocData'; import {nativeCompare} from 'app/common/gutil'; import {IRefCountSub, RefCountMap} from 'app/common/RefCountMap'; import {TableData as BaseTableData} from 'app/common/TableData'; +import {RowFilterFunc} from 'app/common/RowFilterFunc'; import {tbind} from 'app/common/tbind'; import {Disposable, Holder, IDisposableOwnerT} from 'grainjs'; import * as ko from 'knockout'; @@ -295,7 +296,7 @@ export class TableQuerySets { /** * Returns a filtering function which tells whether a row matches the given query. */ -export function getFilterFunc(docData: DocData, query: Query): FilterFunc { +export function getFilterFunc(docData: DocData, query: Query): RowFilterFunc { // NOTE we rely without checking on tableId and colIds being valid. const tableData: BaseTableData = docData.getTable(query.tableId)!; const colIds = Object.keys(query.filters).sort(); diff --git a/app/client/models/SectionFilter.ts b/app/client/models/SectionFilter.ts index 9d424183..d2c1fe24 100644 --- a/app/client/models/SectionFilter.ts +++ b/app/client/models/SectionFilter.ts @@ -1,22 +1,17 @@ import {KoArray} from 'app/client/lib/koArray'; import {ViewFieldRec} from 'app/client/models/DocModel'; -import {FilterFunc, RowId} from 'app/client/models/rowset'; +import {RowId} from 'app/client/models/rowset'; import {TableData} from 'app/client/models/TableData'; -import {CellValue} from 'app/common/DocActions'; import {Computed, Disposable, MutableObsArray, obsArray, Observable} from 'grainjs'; -import {ColumnFilter, ColumnFilterFunc, getFilterFunc} from './ColumnFilter'; - -type RowValueFunc = (rowId: RowId) => CellValue; +import {ColumnFilter} from './ColumnFilter'; +import {buildRowFilter, RowFilterFunc, RowValueFunc } from "app/common/RowFilterFunc"; +import {buildColFilter} from "app/common/ColumnFilterFunc"; interface OpenColumnFilter { fieldRef: number; colFilter: ColumnFilter; } -function buildColFunc(getter: RowValueFunc, filterFunc: ColumnFilterFunc): FilterFunc { - return (rowId: RowId) => filterFunc(getter(rowId)); -} - /** * SectionFilter represents a collection of column filters in place for a view section. It is created * out of `viewFields` and `tableData`, and provides a Computed `sectionFilterFunc` that users can @@ -28,7 +23,7 @@ function buildColFunc(getter: RowValueFunc, filterFunc: ColumnFilterFunc): Filte * results in their being displayed (obviating the need to maintain their rowId explicitly). */ export class SectionFilter extends Disposable { - public readonly sectionFilterFunc: Observable; + public readonly sectionFilterFunc: Observable>; private _openFilterOverride: Observable = Observable.create(this, null); private _tempRows: MutableObsArray = obsArray(); @@ -38,16 +33,16 @@ export class SectionFilter extends Disposable { const columnFilterFunc = Computed.create(this, this._openFilterOverride, (use, openFilter) => { const fields = use(use(viewFields).getObservable()); - const funcs: Array = fields.map(f => { + const funcs: Array | null> = fields.map(f => { const filterFunc = (openFilter && openFilter.fieldRef === f.getRowId()) ? use(openFilter.colFilter.filterFunc) : - getFilterFunc(use(f.activeFilter)); + buildColFilter(use(f.activeFilter)); const getter = tableData.getRowPropFunc(use(f.colId)); if (!filterFunc || !getter) { return null; } - return buildColFunc(getter as RowValueFunc, filterFunc); + return buildRowFilter(getter as RowValueFunc, filterFunc); }) .filter(f => f !== null); // Filter out columns that don't have a filter diff --git a/app/client/models/rowset.ts b/app/client/models/rowset.ts index c7c72e32..d821617c 100644 --- a/app/client/models/rowset.ts +++ b/app/client/models/rowset.ts @@ -25,6 +25,7 @@ import koArray, {KoArray} from 'app/client/lib/koArray'; import {DisposableWithEvents} from 'app/common/DisposableWithEvents'; import {CompareFunc, sortedIndex} from 'app/common/gutil'; import {SkippableRows} from 'app/common/TableData'; +import {RowFilterFunc} from "app/common/RowFilterFunc"; /** * Special constant value that can be used for the `rows` array for the 'rowNotify' @@ -206,8 +207,6 @@ export class ExtendedRowSource extends RowSource { // FilteredRowSource // ---------------------------------------------------------------------- -export type FilterFunc = (row: RowId) => boolean; - interface FilterRowChanges { adds?: RowId[]; updates?: RowId[]; @@ -221,7 +220,7 @@ interface FilterRowChanges { export class BaseFilteredRowSource extends RowListener implements RowSource { protected _matchingRows: Set = new Set(); // Set of rows matching the filter. - constructor(protected _filterFunc: FilterFunc) { + constructor(protected _filterFunc: RowFilterFunc) { super(); } @@ -327,7 +326,7 @@ export class FilteredRowSource extends BaseFilteredRowSource { * Change the filter function. This may trigger 'remove' and 'add' events as necessary to indicate * that rows stopped or started matching the new filter. */ - public updateFilter(filterFunc: FilterFunc) { + public updateFilter(filterFunc: RowFilterFunc) { this._filterFunc = filterFunc; const changes: FilterRowChanges = {}; // After the first call, _excludedRows may have additional rows, but there is no harm in it, diff --git a/app/client/ui/ColumnFilterMenu.ts b/app/client/ui/ColumnFilterMenu.ts index 7ad8564f..76415e17 100644 --- a/app/client/ui/ColumnFilterMenu.ts +++ b/app/client/ui/ColumnFilterMenu.ts @@ -4,7 +4,7 @@ * but on Cancel the model is reset to its initial state prior to menu closing. */ -import {allInclusive, ColumnFilter, isEquivalentFilter} from 'app/client/models/ColumnFilter'; +import {allInclusive, ColumnFilter} from 'app/client/models/ColumnFilter'; import {ColumnFilterMenuModel, IFilterCount} from 'app/client/models/ColumnFilterMenuModel'; import {ViewFieldRec, ViewSectionRec} from 'app/client/models/DocModel'; import {FilteredRowSource} from 'app/client/models/rowset'; @@ -20,6 +20,7 @@ import {Computed, Disposable, dom, DomElementMethod, IDisposableOwner, input, ma import identity = require('lodash/identity'); import noop = require('lodash/noop'); import {IOpenController, IPopupOptions, setPopupToCreateDom} from 'popweasel'; +import {isEquivalentFilter} from "app/common/FilterState"; interface IFilterMenuOptions { model: ColumnFilterMenuModel; diff --git a/app/client/ui/ViewLayoutMenu.ts b/app/client/ui/ViewLayoutMenu.ts index fb9e4f1e..7b78f977 100644 --- a/app/client/ui/ViewLayoutMenu.ts +++ b/app/client/ui/ViewLayoutMenu.ts @@ -1,15 +1,18 @@ import {allCommands} from 'app/client/components/commands'; import {ViewRec, ViewSectionRec} from 'app/client/models/DocModel'; import {testId} from 'app/client/ui2018/cssVars'; -import {menuDivider, menuItemCmd} from 'app/client/ui2018/menus'; +import {menuDivider, menuItemCmd, menuItemLink} from 'app/client/ui2018/menus'; import {dom} from 'grainjs'; /** * Returns a list of menu items for a view section. */ export function makeViewLayoutMenu(viewModel: ViewRec, viewSection: ViewSectionRec, isReadonly: boolean) { + const gristDoc = viewSection.viewInstance.peek()!.gristDoc; return [ menuItemCmd(allCommands.printSection, 'Print widget', testId('print-section')), + menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}, + 'Download as CSV', testId('download-section')), dom.maybe((use) => ['detail', 'single'].includes(use(viewSection.parentKey)), () => menuItemCmd(allCommands.editLayout, 'Edit Card Layout', dom.cls('disabled', isReadonly))), diff --git a/app/common/ColumnFilterFunc.ts b/app/common/ColumnFilterFunc.ts new file mode 100644 index 00000000..7f2d0e91 --- /dev/null +++ b/app/common/ColumnFilterFunc.ts @@ -0,0 +1,18 @@ +import { CellValue } from "app/common/DocActions"; +import { FilterState, makeFilterState } from "app/common/FilterState"; + +export type ColumnFilterFunc = (value: CellValue) => boolean; + +// Returns a filter function for a particular column: the function takes a cell value and returns +// whether it's accepted according to the given FilterState. +export function makeFilterFunc({ include, values }: FilterState): ColumnFilterFunc { + // NOTE: This logic results in complex values and their stringified JSON representations as equivalent. + // For example, a TypeError in the formula column and the string '["E","TypeError"]' would be seen as the same. + // TODO: This narrow corner case seems acceptable for now, but may be worth revisiting. + return (val: CellValue) => (values.has(Array.isArray(val) ? JSON.stringify(val) : val) === include); +} + +// Given a JSON string, returns a ColumnFilterFunc +export function buildColFilter(filterJson: string | undefined): ColumnFilterFunc | null { + return filterJson ? makeFilterFunc(makeFilterState(filterJson)) : null; +} diff --git a/app/common/FilterState.ts b/app/common/FilterState.ts new file mode 100644 index 00000000..bf058b01 --- /dev/null +++ b/app/common/FilterState.ts @@ -0,0 +1,33 @@ +import { CellValue } from "app/common/DocActions"; + +// Filter object as stored in the db +export interface FilterSpec { + included?: CellValue[]; + excluded?: CellValue[]; +} + +// A more efficient representation of filter state for a column than FilterSpec. +export interface FilterState { + include: boolean; + values: Set; +} + +// Creates a FilterState. Accepts spec as a json string or a FilterSpec. +export function makeFilterState(spec: string | FilterSpec): FilterState { + if (typeof (spec) === 'string') { + return makeFilterState((spec && JSON.parse(spec)) || {}); + } + return { + include: Boolean(spec.included), + values: new Set(spec.included || spec.excluded || []), + }; +} + +// Returns true if state and spec are equivalent, false otherwise. +export function isEquivalentFilter(state: FilterState, spec: FilterSpec): boolean { + const other = makeFilterState(spec); + if (state.include !== other.include) { return false; } + if (state.values.size !== other.values.size) { return false; } + for (const val of other.values) { if (!state.values.has(val)) { return false; } } + return true; +} diff --git a/app/common/RowFilterFunc.ts b/app/common/RowFilterFunc.ts new file mode 100644 index 00000000..e199584f --- /dev/null +++ b/app/common/RowFilterFunc.ts @@ -0,0 +1,16 @@ +import { CellValue } from "app/common/DocActions"; +import { ColumnFilterFunc } from "app/common/ColumnFilterFunc"; + +export type RowFilterFunc = (row: T) => boolean; + +// Builds RowFilter for a single column +export function buildRowFilter( + getter: RowValueFunc | null, + filterFunc: ColumnFilterFunc | null): RowFilterFunc { + if (!getter || !filterFunc) { + return () => true; + } + return (rowId: T) => filterFunc(getter(rowId)); +} + +export type RowValueFunc = (rowId: T) => CellValue; diff --git a/app/server/declarations.d.ts b/app/server/declarations.d.ts index 025fd10f..376068da 100644 --- a/app/server/declarations.d.ts +++ b/app/server/declarations.d.ts @@ -1,7 +1,6 @@ declare module "app/server/lib/ActionLog"; declare module "app/server/lib/sandboxUtil"; declare module "app/server/lib/User"; -declare module "app/server/serverMethods"; declare module "app/server/lib/Comm" { import {Client, ClientMethod} from "app/server/lib/Client"; @@ -99,3 +98,5 @@ declare module '@gristlabs/pidusage' { import * as pidusage from 'pidusage'; export = pidusage; } + +declare module "csv"; diff --git a/app/server/lib/DocWorker.ts b/app/server/lib/DocWorker.ts index 39ed5994..1b6a9b45 100644 --- a/app/server/lib/DocWorker.ts +++ b/app/server/lib/DocWorker.ts @@ -30,8 +30,8 @@ export class DocWorker { this._comm = comm; } - public getCSV(req: express.Request, res: express.Response): void { - return generateCSV(req, res, this._comm); + public async getCSV(req: express.Request, res: express.Response): Promise { + await generateCSV(req, res, this._comm); } public async getAttachment(req: express.Request, res: express.Response): Promise { diff --git a/app/server/serverMethods.js b/app/server/serverMethods.js deleted file mode 100644 index 0d3d93a3..00000000 --- a/app/server/serverMethods.js +++ /dev/null @@ -1,111 +0,0 @@ -const gutil = require('app/common/gutil'); -const {SortFunc} = require('app/common/SortFunc'); -const ValueFormatter = require('app/common/ValueFormatter'); -const {docSessionFromRequest} = require('app/server/lib/DocSession'); -const Promise = require('bluebird'); -const contentDisposition = require('content-disposition'); -const csv = require('csv'); -const fs = require('fs-extra'); -const log = require('./lib/log'); -const {ServerColumnGetters} = require('./lib/ServerColumnGetters'); -const multiparty = require('multiparty'); -const tmp = require('tmp'); -const _ = require('underscore'); - -Promise.promisifyAll(csv); -Promise.promisifyAll(multiparty, {filter: name => (name === 'parse'), multiArgs: true}); -Promise.promisifyAll(fs); -Promise.promisifyAll(tmp); - -function generateCSV(req, res, comm) { - log.info('Generating .csv file...'); - // Get the current table id - var tableId = req.param('tableId'); - var viewSectionId = parseInt(req.param('viewSection'), 10); - var activeSortOrder = gutil.safeJsonParse(req.param('activeSortSpec'), null); - - // Get the active doc - var clientId = req.param('clientId'); - var docFD = parseInt(req.param('docFD'), 10); - var client = comm.getClient(clientId); - var docSession = client.getDocSession(docFD); - var activeDoc = docSession.activeDoc; - - // Generate a decent name for the exported file. - var docName = req.query.title || activeDoc.docName; - var name = docName + - (tableId === docName ? '' : '-' + tableId) + '.csv'; - - res.set('Content-Type', 'text/csv'); - res.setHeader('Content-Disposition', contentDisposition(name)); - return makeCSV(activeDoc, viewSectionId, activeSortOrder, req) - .then(data => res.send(data)); -} -exports.generateCSV = generateCSV; - -/** - * Returns a csv stream that can be transformed or parsed. See https://github.com/wdavidw/node-csv - * for API details. - * - * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. - * @param {Integer} viewSectionId - id of the viewsection to export. - * @param {Integer[]} activeSortOrder (optional) - overriding sort order. - * @return {Promise} Promise for the resulting CSV. - */ -function makeCSV(activeDoc, viewSectionId, sortOrder, req) { - return Promise.try(() => { - const tables = activeDoc.docData.getTables(); - const viewSection = tables.get('_grist_Views_section').getRecord(viewSectionId); - const table = tables.get('_grist_Tables').getRecord(viewSection.tableRef); - const fields = tables.get('_grist_Views_section_field').filterRecords({ parentId: viewSection.id}); - const tableColumns = tables.get('_grist_Tables_column').filterRecords({parentId: table.id}); - const tableColsById = _.indexBy(tableColumns, 'id'); - - // Produce a column description matching what user will see / expect to export - const viewify = (col, field) => { - field = field || {}; - const displayCol = tableColsById[field.displayCol || col.displayCol || col.id]; - const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {}); - const fieldWidgetOptions = gutil.safeJsonParse(field.widgetOptions, {}); - return { - id: displayCol.id, - colId: displayCol.colId, - label: col.label, - colType: col.type, - widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions) - }; - }; - const viewColumns = _.sortBy(fields, 'parentPos').map( - (field) => viewify(tableColsById[field.colRef], field)); - - // The columns named in sort order need to now become display columns - sortOrder = sortOrder || gutil.safeJsonParse(viewSection.sortColRefs, []); - const fieldsByColRef = _.indexBy(fields, 'colRef'); - sortOrder = sortOrder.map((directionalColRef) => { - const colRef = Math.abs(directionalColRef); - const col = tableColsById[colRef]; - if (!col) return 0; - const effectiveColRef = viewify(col, fieldsByColRef[colRef]).id; - return directionalColRef > 0 ? effectiveColRef : -effectiveColRef; - }); - - return [activeDoc.fetchTable(docSessionFromRequest(req), table.tableId, true), tableColumns, viewColumns]; - }).spread((data, tableColumns, viewColumns) => { - const rowIds = data[2]; - const dataByColId = data[3]; - const getters = new ServerColumnGetters(rowIds, dataByColId, tableColumns); - const sorter = new SortFunc(getters); - sorter.updateSpec(sortOrder); - rowIds.sort((a, b) => sorter.compare(a, b)); - const formatters = viewColumns.map(col => - ValueFormatter.createFormatter(col.colType, col.widgetOptions)); - // Arrange the data into a row-indexed matrix, starting with column headers. - const csvMatrix = [viewColumns.map(col => col.label)]; - const access = viewColumns.map(col => getters.getColGetter(col.id)); - rowIds.forEach(row => { - csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter(row)))); - }); - return csv.stringifyAsync(csvMatrix); - }); -} -exports.makeCSV = makeCSV; diff --git a/app/server/serverMethods.ts b/app/server/serverMethods.ts new file mode 100644 index 00000000..60a19ea9 --- /dev/null +++ b/app/server/serverMethods.ts @@ -0,0 +1,209 @@ +import * as gutil from 'app/common/gutil'; +import { SortFunc } from "app/common/SortFunc"; +import { docSessionFromRequest } from "app/server/lib/DocSession"; +import * as bluebird from "bluebird"; +import * as contentDisposition from "content-disposition"; +import * as csv from "csv"; +import * as log from "./lib/log"; +import { ServerColumnGetters } from "./lib/ServerColumnGetters"; +import * as _ from "underscore"; +import * as express from "express"; +import * as Comm from 'app/server/lib/Comm'; +import { ActiveDoc } from "app/server/lib/ActiveDoc"; +import { createFormatter } from "app/common/ValueFormatter"; +import { SchemaTypes } from "app/common/schema"; +import { RequestWithLogin } from "app/server/lib/Authorizer"; +import { RowRecord } from "app/common/DocActions"; +import { buildColFilter } from "app/common/ColumnFilterFunc"; +import { buildRowFilter } from "app/common/RowFilterFunc"; + +// promisify csv +bluebird.promisifyAll(csv); + +export async function generateCSV(req: express.Request, res: express.Response, comm: Comm) { + + log.info('Generating .csv file...'); + // Get the current table id + const tableId = req.param('tableId'); + const viewSectionId = parseInt(req.param('viewSection'), 10); + const activeSortOrder = gutil.safeJsonParse(req.param('activeSortSpec'), null); + const filters: Filter[] = gutil.safeJsonParse(req.param("filters"), []) || []; + + // Get the active doc + const clientId = req.param('clientId'); + const docFD = parseInt(req.param('docFD'), 10); + const client = comm.getClient(clientId); + const docSession = client.getDocSession(docFD); + const activeDoc = docSession.activeDoc; + + // Generate a decent name for the exported file. + const docName = req.query.title || activeDoc.docName; + const name = docName + + (tableId === docName ? '' : '-' + tableId) + '.csv'; + + try { + const data = await makeCSV(activeDoc, viewSectionId, activeSortOrder, filters, req); + res.set('Content-Type', 'text/csv'); + res.setHeader('Content-Disposition', contentDisposition(name)); + res.send(data); + } catch (err) { + log.error("Exporting to CSV has failed. Request url: %s", req.url, err); + // send a generic information to client + const errHtml = + ` + + There was an unexpected error while generating a csv file. + +`; + res.status(400).send(errHtml); + } +} + +/** + * Returns a csv stream that can be transformed or parsed. See https://github.com/wdavidw/node-csv + * for API details. + * + * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. + * @param {Integer} viewSectionId - id of the viewsection to export. + * @param {Integer[]} activeSortOrder (optional) - overriding sort order. + * @return {Promise} Promise for the resulting CSV. + */ +export async function makeCSV( + activeDoc: ActiveDoc, + viewSectionId: number, + sortOrder: number[], + filters: Filter[], + req: express.Request) { + + const { + table, + viewSection, + tableColumns, + fields + } = explodeSafe(activeDoc, viewSectionId); + + const tableColsById = _.indexBy(tableColumns, 'id'); + + // Produce a column description matching what user will see / expect to export + const viewify = (col: GristTablesColumn, field: GristViewsSectionField) => { + field = field || {}; + const displayCol = tableColsById[field.displayCol || col.displayCol || col.id]; + const colWidgetOptions = gutil.safeJsonParse(col.widgetOptions, {}); + const fieldWidgetOptions = gutil.safeJsonParse(field.widgetOptions, {}); + const filterFunc = buildColFilter(filters.find(x => x.colRef === field.colRef)?.filter); + return { + id: displayCol.id, + colId: displayCol.colId, + label: col.label, + colType: col.type, + filterFunc, + widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions) + }; + }; + const viewColumns = _.sortBy(fields, 'parentPos').map( + (field) => viewify(tableColsById[field.colRef], field)); + + // The columns named in sort order need to now become display columns + sortOrder = sortOrder || gutil.safeJsonParse(viewSection.sortColRefs, []); + const fieldsByColRef = _.indexBy(fields, 'colRef'); + sortOrder = sortOrder.map((directionalColRef) => { + const colRef = Math.abs(directionalColRef); + const col = tableColsById[colRef]; + if (!col) { + return 0; + } + const effectiveColRef = viewify(col, fieldsByColRef[colRef]).id; + return directionalColRef > 0 ? effectiveColRef : -effectiveColRef; + }); + + const data = await activeDoc.fetchTable(docSessionFromRequest(req as RequestWithLogin), table.tableId, true); + const rowIds = data[2]; + const dataByColId = data[3]; + const getters = new ServerColumnGetters(rowIds, dataByColId, tableColumns); + const sorter = new SortFunc(getters); + sorter.updateSpec(sortOrder); + rowIds.sort((a, b) => sorter.compare(a, b)); + const formatters = viewColumns.map(col => + createFormatter(col.colType, col.widgetOptions)); + // Arrange the data into a row-indexed matrix, starting with column headers. + const csvMatrix = [viewColumns.map(col => col.label)]; + const access = viewColumns.map(col => getters.getColGetter(col.id)); + // create row filter based on all columns filter + const rowFilter = viewColumns + .map((col, c) => buildRowFilter(access[c], col.filterFunc)) + .reduce((prevFilter, curFilter) => (id) => prevFilter(id) && curFilter(id), () => true); + rowIds.forEach(row => { + if (!rowFilter(row)) { + return; + } + csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter!(row)))); + }); + + return csv.stringifyAsync(csvMatrix); +} + + +// helper method that retrieves various parts about view section +// from ActiveDoc +function explodeSafe(activeDoc: ActiveDoc, viewSectionId: number) { + const docData = activeDoc.docData; + + if (!docData) { + // Should not happen unless there's a logic error + // This method is exported (for testing) so it is possible + // to call it without loading active doc first + throw new Error("Document hasn't been loaded yet"); + } + + const viewSection = docData + .getTable('_grist_Views_section') + ?.getRecord(viewSectionId) as GristViewsSection | undefined; + + if (!viewSection) { + throw new Error(`No table '_grist_Views_section' in document with id ${activeDoc.docName}`); + } + + const table = docData + .getTable('_grist_Tables') + ?.getRecord(viewSection.tableRef) as GristTables | undefined; + + if (!table) { + throw new Error(`No table '_grist_Tables' in document with id ${activeDoc.docName}`); + } + + const fields = docData + .getTable('_grist_Views_section_field') + ?.filterRecords({ parentId: viewSection.id }) as GristViewsSectionField[] | undefined; + + if (!fields) { + throw new Error(`No table '_grist_Views_section_field' in document with id ${activeDoc.docName}`); + } + + const tableColumns = docData + .getTable('_grist_Tables_column') + ?.filterRecords({ parentId: table.id }) as GristTablesColumn[] | undefined; + + if (!tableColumns) { + throw new Error(`No table '_grist_Tables_column' in document with id ${activeDoc.docName}`); + } + + return { + table, + fields, + tableColumns, + viewSection + }; +} + + +// Type helpers for types used in this export +type RowModel = RowRecord & { + [ColId in keyof SchemaTypes[TName]]: SchemaTypes[TName][ColId]; +}; +type GristViewsSection = RowModel<'_grist_Views_section'> +type GristTables = RowModel<'_grist_Tables'> +type GristViewsSectionField = RowModel<'_grist_Views_section_field'> +type GristTablesColumn = RowModel<'_grist_Tables_column'> + +// Type for filters passed from the client +interface Filter { colRef: number, filter: string }