(core) Add TSV and DSV import/export

Summary: Adds support for importing .dsv files (an April Fools 2024 easter egg), and options for exporting .dsv and .tsv files from the Share menu.

Test Plan: Browser and server tests.

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D4210
This commit is contained in:
George Gevoian 2024-03-20 09:58:24 -04:00
parent 48a8af83fc
commit 07fcce548b
15 changed files with 2672 additions and 45 deletions

View File

@ -1222,6 +1222,16 @@ export class GristDoc extends DisposableWithEvents {
return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadCsvUrl(params);
}
public getTsvLink() {
const params = this._getDocApiDownloadParams();
return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadTsvUrl(params);
}
public getDsvLink() {
const params = this._getDocApiDownloadParams();
return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadDsvUrl(params);
}
public getXlsxActiveViewLink() {
const params = this._getDocApiDownloadParams();
return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadXlsxUrl(params);

View File

@ -31,7 +31,7 @@ export interface SelectFileOptions extends UploadOptions {
// e.g. [".jpg", ".png"]
}
export const IMPORTABLE_EXTENSIONS = [".grist", ".csv", ".tsv", ".txt", ".xlsx", ".xlsm"];
export const IMPORTABLE_EXTENSIONS = [".grist", ".csv", ".tsv", ".dsv", ".txt", ".xlsx", ".xlsm"];
/**
* Shows the file-picker dialog with the given options, and uploads the selected files. If under

View File

@ -10,7 +10,8 @@ import {cssHoverCircle, cssTopBarBtn} from 'app/client/ui/TopBarCss';
import {primaryButton} from 'app/client/ui2018/buttons';
import {mediaXSmall, testId, theme} from 'app/client/ui2018/cssVars';
import {icon} from 'app/client/ui2018/icons';
import {menu, menuAnnotate, menuDivider, menuIcon, menuItem, menuItemLink, menuText} from 'app/client/ui2018/menus';
import {menu, menuAnnotate, menuDivider, menuIcon, menuItem, menuItemLink, menuItemSubmenu,
menuText} from 'app/client/ui2018/menus';
import {buildUrlId, isFeatureEnabled, parseUrlId} from 'app/common/gristUrls';
import * as roles from 'app/common/roles';
import {Document} from 'app/common/UserAPI';
@ -262,7 +263,7 @@ function menuWorkOnCopy(pageModel: DocPageModel) {
}
/**
* The part of the menu with "Download" and "Export CSV" items.
* The part of the menu with "Download" and "Export as..." items.
*/
function menuExports(doc: Document, pageModel: DocPageModel) {
const isElectron = (window as any).isRunningUnderElectron;
@ -278,12 +279,24 @@ function menuExports(doc: Document, pageModel: DocPageModel) {
menuItem(() => downloadDocModal(doc, pageModel),
menuIcon('Download'), t("Download..."), testId('tb-share-option'))
),
menuItemSubmenu(
() => [
menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}),
menuIcon('Download'), t("Export CSV"), testId('tb-share-option')),
t("Comma Separated Values (.csv)"), testId('tb-share-option')),
menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getTsvLink(), target: '_blank', download: ''}),
t("Tab Separated Values (.tsv)"), testId('tb-share-option')),
menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getDsvLink(), target: '_blank', download: ''}),
t("DOO Separated Values (.dsv)"), testId('tb-share-option')),
menuItemLink(hooks.maybeModifyLinkAttrs({
href: pageModel.appModel.api.getDocAPI(doc.id).getDownloadXlsxUrl(),
target: '_blank', download: ''
}), menuIcon('Download'), t("Export XLSX"), testId('tb-share-option')),
}), t("Microsoft Excel (.xlsx)"), testId('tb-share-option')),
],
{},
menuIcon('Download'),
t("Export as..."),
testId('tb-share-option'),
),
(!isFeatureEnabled("sendToDrive") ? null : menuItem(() => sendToDrive(doc, pageModel),
menuIcon('Download'), t("Send to Google Drive"), testId('tb-share-option'))),
];

View File

@ -92,8 +92,13 @@ export interface TransformColumn {
widgetOptions: string;
}
export interface ImportParseOptions extends ParseOptions {
delimiter?: string;
encoding?: string;
}
export interface ImportResult {
options: ParseOptions;
options: ImportParseOptions;
tables: ImportTableResult[];
}
@ -106,7 +111,7 @@ export interface ImportTableResult {
}
export interface ImportOptions {
parseOptions?: ParseOptions; // Options for parsing the source file.
parseOptions?: ImportParseOptions; // Options for parsing the source file.
mergeOptionMaps?: MergeOptionsMap[]; // Options for merging fields, indexed by uploadFileIndex.
}
@ -328,7 +333,7 @@ export interface ActiveDocAPI {
* Imports files, removes previously created temporary hidden tables and creates the new ones.
*/
importFiles(dataSource: DataSourceTransformed,
parseOptions: ParseOptions, prevTableIds: string[]): Promise<ImportResult>;
parseOptions: ImportParseOptions, prevTableIds: string[]): Promise<ImportResult>;
/**
* Finishes import files, creates the new tables, and cleans up temporary hidden tables and uploads.

View File

@ -470,6 +470,8 @@ export interface DocAPI {
getDownloadUrl(options: {template: boolean, removeHistory: boolean}): string;
getDownloadXlsxUrl(params?: DownloadDocParams): string;
getDownloadCsvUrl(params: DownloadDocParams): string;
getDownloadTsvUrl(params: DownloadDocParams): string;
getDownloadDsvUrl(params: DownloadDocParams): string;
getDownloadTableSchemaUrl(params: DownloadDocParams): string;
/**
* Exports current document to the Google Drive as a spreadsheet file. To invoke this method, first
@ -1057,6 +1059,14 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
return this._url + '/download/csv?' + encodeQueryParams({...params});
}
public getDownloadTsvUrl(params: DownloadDocParams) {
return this._url + '/download/tsv?' + encodeQueryParams({...params});
}
public getDownloadDsvUrl(params: DownloadDocParams) {
return this._url + '/download/dsv?' + encodeQueryParams({...params});
}
public getDownloadTableSchemaUrl(params: DownloadDocParams) {
// We spread `params` to work around TypeScript being overly cautious.
return this._url + '/download/table-schema?' + encodeQueryParams({...params});

View File

@ -345,8 +345,17 @@ export class ActiveDocImport {
if (file.ext) {
origName = path.basename(origName, path.extname(origName)) + file.ext;
}
const fileParseOptions = {...parseOptions};
if (file.ext === '.dsv') {
if (!fileParseOptions.delimiter) {
fileParseOptions.delimiter = '💩';
}
if (!fileParseOptions.encoding) {
fileParseOptions.encoding = 'utf-8';
}
}
const res = await this._importFileAsNewTable(docSession, file.absPath, {
parseOptions,
parseOptions: fileParseOptions,
mergeOptionsMap: mergeOptionMaps[index] || {},
isHidden,
originalFilename: origName,

View File

@ -53,7 +53,7 @@ import {docSessionFromRequest, getDocSessionShare, makeExceptionalDocSession,
import {DocWorker} from "app/server/lib/DocWorker";
import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
import {DownloadOptions, parseExportParameters} from "app/server/lib/Export";
import {downloadCSV} from "app/server/lib/ExportCSV";
import {downloadDSV} from "app/server/lib/ExportDSV";
import {collectTableSchemaInFrictionlessFormat} from "app/server/lib/ExportTableSchema";
import {streamXLSX} from "app/server/lib/ExportXLSX";
import {expressWrap} from 'app/server/lib/expressWrap';
@ -1247,7 +1247,19 @@ export class DocWorkerApi {
this._app.get('/api/docs/:docId/download/csv', canView, withDoc(async (activeDoc, req, res) => {
const options = await this._getDownloadOptions(req);
await downloadCSV(activeDoc, req, res, options);
await downloadDSV(activeDoc, req, res, {...options, delimiter: ','});
}));
this._app.get('/api/docs/:docId/download/tsv', canView, withDoc(async (activeDoc, req, res) => {
const options = await this._getDownloadOptions(req);
await downloadDSV(activeDoc, req, res, {...options, delimiter: '\t'});
}));
this._app.get('/api/docs/:docId/download/dsv', canView, withDoc(async (activeDoc, req, res) => {
const options = await this._getDownloadOptions(req);
await downloadDSV(activeDoc, req, res, {...options, delimiter: '💩'});
}));
this._app.get('/api/docs/:docId/download/xlsx', canView, withDoc(async (activeDoc, req, res) => {

View File

@ -146,6 +146,8 @@ export class DocPluginManager {
'.xlsx' : 'Excel',
'.json' : 'JSON',
'.csv' : 'CSV',
'.tsv' : 'TSV',
'.dsv' : 'PSV',
};
const fileType = extToType[path.extname(fileName)] || path.extname(fileName);
throw new Error(`Failed to parse ${fileType} file.\nError: ${messages.join("; ")}`);

View File

@ -11,26 +11,38 @@ import * as express from 'express';
// promisify csv
bluebird.promisifyAll(csv);
export interface DownloadDsvOptions extends DownloadOptions {
delimiter: Delimiter;
}
type Delimiter = ',' | '\t' | '💩';
/**
* Converts `activeDoc` to a CSV and sends the converted data through `res`.
* Converts `activeDoc` to delimiter-separated values (e.g. CSV) and sends
* the converted data through `res`.
*/
export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
res: express.Response, options: DownloadOptions) {
log.info('Generating .csv file...');
const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, header} = options;
export async function downloadDSV(
activeDoc: ActiveDoc,
req: express.Request,
res: express.Response,
options: DownloadDsvOptions
) {
const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, delimiter, header} = options;
const extension = getDSVFileExtension(delimiter);
log.info(`Generating ${extension} file...`);
const data = viewSectionId ?
await makeCSVFromViewSection({
await makeDSVFromViewSection({
activeDoc, viewSectionId, sortOrder: sortOrder || null, filters: filters || null,
linkingFilter: linkingFilter || null, header, req
linkingFilter: linkingFilter || null, header, delimiter, req
}) :
await makeCSVFromTable({activeDoc, tableId, header, req});
res.set('Content-Type', 'text/csv');
res.setHeader('Content-Disposition', contentDisposition(filename + '.csv'));
await makeDSVFromTable({activeDoc, tableId, header, delimiter, req});
res.set('Content-Type', getDSVMimeType(delimiter));
res.setHeader('Content-Disposition', contentDisposition(filename + extension));
res.send(data);
}
/**
* Returns a csv stream of a view section that can be transformed or parsed.
* Returns a DSV stream of a view section that can be transformed or parsed.
*
* See https://github.com/wdavidw/node-csv for API details.
*
@ -40,13 +52,21 @@ export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
* @param {Integer[]} options.activeSortOrder (optional) - overriding sort order.
* @param {Filter[]} options.filters (optional) - filters defined from ui.
* @param {FilterColValues} options.linkingFilter (optional) - linking filter defined from ui.
* @param {Delimiter} options.delimiter - delimiter to separate fields with
* @param {string} options.header (optional) - which field of the column to use as header
* @param {express.Request} options.req - the request object.
*
* @return {Promise<string>} Promise for the resulting CSV.
* @return {Promise<string>} Promise for the resulting DSV.
*/
export async function makeCSVFromViewSection({
activeDoc, viewSectionId, sortOrder = null, filters = null, linkingFilter = null, header, req
export async function makeDSVFromViewSection({
activeDoc,
viewSectionId,
sortOrder = null,
filters = null,
linkingFilter = null,
delimiter,
header,
req
}: {
activeDoc: ActiveDoc,
viewSectionId: number,
@ -54,28 +74,31 @@ export async function makeCSVFromViewSection({
filters: Filter[] | null,
linkingFilter: FilterColValues | null,
header?: ExportHeader,
delimiter: Delimiter,
req: express.Request
}) {
const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, linkingFilter, req);
const file = convertToCsv(data, { header });
const file = convertToDsv(data, { header, delimiter });
return file;
}
/**
* Returns a csv stream of a table that can be transformed or parsed.
* Returns a DSV stream of a table that can be transformed or parsed.
*
* @param {Object} options - options for the export.
* @param {Object} options.activeDoc - the activeDoc that the table being converted belongs to.
* @param {Integer} options.tableId - id of the table to export.
* @param {Delimiter} options.delimiter - delimiter to separate fields with
* @param {string} options.header (optional) - which field of the column to use as header
* @param {express.Request} options.req - the request object.
*
* @return {Promise<string>} Promise for the resulting CSV.
* @return {Promise<string>} Promise for the resulting DSV.
*/
export async function makeCSVFromTable({ activeDoc, tableId, header, req }: {
export async function makeDSVFromTable({ activeDoc, tableId, delimiter, header, req }: {
activeDoc: ActiveDoc,
tableId: string,
delimiter: Delimiter,
header?: ExportHeader,
req: express.Request
}) {
@ -93,24 +116,63 @@ export async function makeCSVFromTable({ activeDoc, tableId, header, req }: {
}
const data = await exportTable(activeDoc, tableRef, req);
const file = convertToCsv(data, { header });
const file = convertToDsv(data, { header, delimiter });
return file;
}
function convertToCsv({
rowIds,
access,
columns: viewColumns,
}: ExportData, options: { header?: ExportHeader }) {
interface ConvertToDsvOptions {
delimiter: Delimiter;
header?: ExportHeader;
}
function convertToDsv(data: ExportData, options: ConvertToDsvOptions) {
const {rowIds, access, columns: viewColumns} = data;
const {delimiter, header} = options;
// create formatters for columns
const formatters = viewColumns.map(col => col.formatter);
// Arrange the data into a row-indexed matrix, starting with column headers.
const colPropertyAsHeader = options.header ?? 'label';
const colPropertyAsHeader = header ?? 'label';
const csvMatrix = [viewColumns.map(col => col[colPropertyAsHeader])];
// populate all the rows with values as strings
rowIds.forEach(row => {
csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter(row))));
});
return csv.stringifyAsync(csvMatrix);
return csv.stringifyAsync(csvMatrix, {delimiter});
}
type DSVFileExtension = '.csv' | '.tsv' | '.dsv';
function getDSVFileExtension(delimiter: Delimiter): DSVFileExtension {
switch (delimiter) {
case ',': {
return '.csv';
}
case '\t': {
return '.tsv';
}
case '💩': {
return '.dsv';
}
}
}
type DSVMimeType =
| 'text/csv'
// Reference: https://www.iana.org/assignments/media-types/text/tab-separated-values
| 'text/tab-separated-values'
// Note: not a registered MIME type, hence the "x-" prefix.
| 'text/x-doo-separated-values';
function getDSVMimeType(delimiter: Delimiter): DSVMimeType {
switch (delimiter) {
case ',': {
return 'text/csv';
}
case '\t': {
return 'text/tab-separated-values';
}
case '💩': {
return 'text/x-doo-separated-values';
}
}
}

View File

@ -5,7 +5,7 @@ components:
safePython: sandbox/main.py
contributions:
fileParsers:
- fileExtensions: ["csv", "tsv", "txt"]
- fileExtensions: ["csv", "tsv", "dsv", "txt"]
parseFile:
component: safePython
name: csv_parser

File diff suppressed because it is too large Load Diff

4
test/fixtures/export-dsv/text.dsv vendored Normal file
View File

@ -0,0 +1,4 @@
Foo💩Bar💩Id is Baz Label is this💩Link💩Formula
1💩a💩hello💩grist https://www.getgrist.com/💩a --- grist https://www.getgrist.com/
2💩b ,d💩world💩https://www.getgrist.com/💩b ,d --- https://www.getgrist.com/
3💩"the ""quote marks"" ?"💩💩💩"the ""quote marks"" ? --- "

File diff suppressed because it is too large Load Diff

4
test/fixtures/export-tsv/text.tsv vendored Normal file
View File

@ -0,0 +1,4 @@
Foo Bar Id is Baz Label is this Link Formula
1 a hello grist https://www.getgrist.com/ a --- grist https://www.getgrist.com/
2 b ,d world https://www.getgrist.com/ b ,d --- https://www.getgrist.com/
3 "the ""quote marks"" ?" "the ""quote marks"" ? --- "
1 Foo Bar Id is Baz Label is this Link Formula
2 1 a hello grist https://www.getgrist.com/ a --- grist https://www.getgrist.com/
3 2 b ,d world https://www.getgrist.com/ b ,d --- https://www.getgrist.com/
4 3 the "quote marks" ? the "quote marks" ? ---

View File

@ -1,4 +1,4 @@
import { assert } from 'mocha-webdriver';
import { assert, driver } from 'mocha-webdriver';
import { $, gu, test } from 'test/nbrowser/gristUtil-nbrowser';
const fse = require('fs-extra');
@ -35,7 +35,10 @@ describe('Export.ntest', function() {
await $('.test-tb-share').click();
// Once the menu opens, get the href of the link.
await $('.grist-floating-menu').wait();
const href = await $('.grist-floating-menu a:contains(CSV)').wait().getAttribute('href');
const submenu = $('.test-tb-share-option:contains(Export as...)');
await driver.withActions(a => a.move({origin: submenu.elem()}));
const href = await $('.grist-floating-menu a:contains(Comma Separated Values)').wait()
.getAttribute('href');
// Download the data at the link and compare to expected.
const resp = await axios.get(href, {responseType: 'text', headers});
assert.equal(resp.headers['content-disposition'],
@ -50,7 +53,10 @@ describe('Export.ntest', function() {
await $('.test-tb-share').click();
// Once the menu opens, get the href of the link.
await $('.grist-floating-menu').wait();
const href = await $('.grist-floating-menu a:contains(CSV)').wait().getAttribute('href');
const submenu = $('.test-tb-share-option:contains(Export as...)');
await driver.withActions(a => a.move({origin: submenu.elem()}));
const href = await $('.grist-floating-menu a:contains(Comma Separated Values)').wait()
.getAttribute('href');
// Download the data at the link and compare to expected.
const resp = await axios.get(href, {responseType: 'text', headers});
assert.equal(resp.data, dataExpected.sorted);