(core) Add TSV and DSV import/export

Summary: Adds support for importing .dsv files (an April Fools 2024 easter egg), and options for exporting .dsv and .tsv files from the Share menu.

Test Plan: Browser and server tests.

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D4210

app/client/components/GristDoc.ts

@@ -1222,6 +1222,16 @@ export class GristDoc extends DisposableWithEvents {
     return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadCsvUrl(params);
   }
 
+  public getTsvLink() {
+    const params = this._getDocApiDownloadParams();
+    return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadTsvUrl(params);
+  }
+
+  public getDsvLink() {
+    const params = this._getDocApiDownloadParams();
+    return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadDsvUrl(params);
+  }
+
   public getXlsxActiveViewLink() {
     const params = this._getDocApiDownloadParams();
     return this.docPageModel.appModel.api.getDocAPI(this.docId()).getDownloadXlsxUrl(params);

app/client/lib/uploads.ts

@@ -31,7 +31,7 @@ export interface SelectFileOptions extends UploadOptions {
                           // e.g. [".jpg", ".png"]
 }
 
-export const IMPORTABLE_EXTENSIONS = [".grist", ".csv", ".tsv", ".txt", ".xlsx", ".xlsm"];
+export const IMPORTABLE_EXTENSIONS = [".grist", ".csv", ".tsv", ".dsv", ".txt", ".xlsx", ".xlsm"];
 
 /**
  * Shows the file-picker dialog with the given options, and uploads the selected files. If under

app/client/ui/ShareMenu.ts

@@ -10,7 +10,8 @@ import {cssHoverCircle, cssTopBarBtn} from 'app/client/ui/TopBarCss';
 import {primaryButton} from 'app/client/ui2018/buttons';
 import {mediaXSmall, testId, theme} from 'app/client/ui2018/cssVars';
 import {icon} from 'app/client/ui2018/icons';
-import {menu, menuAnnotate, menuDivider, menuIcon, menuItem, menuItemLink, menuText} from 'app/client/ui2018/menus';
+import {menu, menuAnnotate, menuDivider, menuIcon, menuItem, menuItemLink, menuItemSubmenu,
+        menuText} from 'app/client/ui2018/menus';
 import {buildUrlId, isFeatureEnabled, parseUrlId} from 'app/common/gristUrls';
 import * as roles from 'app/common/roles';
 import {Document} from 'app/common/UserAPI';
@@ -262,7 +263,7 @@ function menuWorkOnCopy(pageModel: DocPageModel) {
 }
 
 /**
- * The part of the menu with "Download" and "Export CSV" items.
+ * The part of the menu with "Download" and "Export as..." items.
  */
 function menuExports(doc: Document, pageModel: DocPageModel) {
   const isElectron = (window as any).isRunningUnderElectron;
@@ -278,12 +279,24 @@ function menuExports(doc: Document, pageModel: DocPageModel) {
         menuItem(() => downloadDocModal(doc, pageModel),
         menuIcon('Download'), t("Download..."), testId('tb-share-option'))
     ),
-    menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}),
-      menuIcon('Download'), t("Export CSV"), testId('tb-share-option')),
-    menuItemLink(hooks.maybeModifyLinkAttrs({
-      href: pageModel.appModel.api.getDocAPI(doc.id).getDownloadXlsxUrl(),
-      target: '_blank', download: ''
-    }), menuIcon('Download'), t("Export XLSX"), testId('tb-share-option')),
+    menuItemSubmenu(
+      () => [
+        menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}),
+          t("Comma Separated Values (.csv)"), testId('tb-share-option')),
+        menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getTsvLink(), target: '_blank', download: ''}),
+          t("Tab Separated Values (.tsv)"), testId('tb-share-option')),
+        menuItemLink(hooks.maybeModifyLinkAttrs({ href: gristDoc.getDsvLink(), target: '_blank', download: ''}),
+          t("DOO Separated Values (.dsv)"), testId('tb-share-option')),
+        menuItemLink(hooks.maybeModifyLinkAttrs({
+          href: pageModel.appModel.api.getDocAPI(doc.id).getDownloadXlsxUrl(),
+          target: '_blank', download: ''
+        }), t("Microsoft Excel (.xlsx)"), testId('tb-share-option')),
+      ],
+      {},
+      menuIcon('Download'),
+      t("Export as..."),
+      testId('tb-share-option'),
+    ),
     (!isFeatureEnabled("sendToDrive") ? null : menuItem(() => sendToDrive(doc, pageModel),
       menuIcon('Download'), t("Send to Google Drive"), testId('tb-share-option'))),
   ];

app/common/ActiveDocAPI.ts

@@ -92,8 +92,13 @@ export interface TransformColumn {
   widgetOptions: string;
 }
 
+export interface ImportParseOptions extends ParseOptions {
+  delimiter?: string;
+  encoding?: string;
+}
+
 export interface ImportResult {
-  options: ParseOptions;
+  options: ImportParseOptions;
   tables: ImportTableResult[];
 }
 
@@ -106,7 +111,7 @@ export interface ImportTableResult {
 }
 
 export interface ImportOptions {
-  parseOptions?: ParseOptions;         // Options for parsing the source file.
+  parseOptions?: ImportParseOptions;   // Options for parsing the source file.
   mergeOptionMaps?: MergeOptionsMap[]; // Options for merging fields, indexed by uploadFileIndex.
 }
 
@@ -328,7 +333,7 @@ export interface ActiveDocAPI {
    * Imports files, removes previously created temporary hidden tables and creates the new ones.
    */
   importFiles(dataSource: DataSourceTransformed,
-              parseOptions: ParseOptions, prevTableIds: string[]): Promise<ImportResult>;
+              parseOptions: ImportParseOptions, prevTableIds: string[]): Promise<ImportResult>;
 
   /**
    * Finishes import files, creates the new tables, and cleans up temporary hidden tables and uploads.

app/common/UserAPI.ts

@@ -470,6 +470,8 @@ export interface DocAPI {
   getDownloadUrl(options: {template: boolean, removeHistory: boolean}): string;
   getDownloadXlsxUrl(params?: DownloadDocParams): string;
   getDownloadCsvUrl(params: DownloadDocParams): string;
+  getDownloadTsvUrl(params: DownloadDocParams): string;
+  getDownloadDsvUrl(params: DownloadDocParams): string;
   getDownloadTableSchemaUrl(params: DownloadDocParams): string;
   /**
    * Exports current document to the Google Drive as a spreadsheet file. To invoke this method, first
@@ -1057,6 +1059,14 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
     return this._url + '/download/csv?' + encodeQueryParams({...params});
   }
 
+  public getDownloadTsvUrl(params: DownloadDocParams) {
+    return this._url + '/download/tsv?' + encodeQueryParams({...params});
+  }
+
+  public getDownloadDsvUrl(params: DownloadDocParams) {
+    return this._url + '/download/dsv?' + encodeQueryParams({...params});
+  }
+
   public getDownloadTableSchemaUrl(params: DownloadDocParams) {
     // We spread `params` to work around TypeScript being overly cautious.
     return this._url + '/download/table-schema?' + encodeQueryParams({...params});

app/server/lib/ActiveDocImport.ts

@@ -345,8 +345,17 @@ export class ActiveDocImport {
       if (file.ext) {
         origName = path.basename(origName, path.extname(origName)) + file.ext;
       }
+      const fileParseOptions = {...parseOptions};
+      if (file.ext === '.dsv') {
+        if (!fileParseOptions.delimiter) {
+          fileParseOptions.delimiter = '💩';
+        }
+        if (!fileParseOptions.encoding) {
+          fileParseOptions.encoding = 'utf-8';
+        }
+      }
       const res = await this._importFileAsNewTable(docSession, file.absPath, {
-        parseOptions,
+        parseOptions: fileParseOptions,
         mergeOptionsMap: mergeOptionMaps[index] || {},
         isHidden,
         originalFilename: origName,

app/server/lib/DocApi.ts

@@ -53,7 +53,7 @@ import {docSessionFromRequest, getDocSessionShare, makeExceptionalDocSession,
 import {DocWorker} from "app/server/lib/DocWorker";
 import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
 import {DownloadOptions, parseExportParameters} from "app/server/lib/Export";
-import {downloadCSV} from "app/server/lib/ExportCSV";
+import {downloadDSV} from "app/server/lib/ExportDSV";
 import {collectTableSchemaInFrictionlessFormat} from "app/server/lib/ExportTableSchema";
 import {streamXLSX} from "app/server/lib/ExportXLSX";
 import {expressWrap} from 'app/server/lib/expressWrap';
@@ -1247,7 +1247,19 @@ export class DocWorkerApi {
     this._app.get('/api/docs/:docId/download/csv', canView, withDoc(async (activeDoc, req, res) => {
       const options = await this._getDownloadOptions(req);
 
-      await downloadCSV(activeDoc, req, res, options);
+      await downloadDSV(activeDoc, req, res, {...options, delimiter: ','});
+    }));
+
+    this._app.get('/api/docs/:docId/download/tsv', canView, withDoc(async (activeDoc, req, res) => {
+      const options = await this._getDownloadOptions(req);
+
+      await downloadDSV(activeDoc, req, res, {...options, delimiter: '\t'});
+    }));
+
+    this._app.get('/api/docs/:docId/download/dsv', canView, withDoc(async (activeDoc, req, res) => {
+      const options = await this._getDownloadOptions(req);
+
+      await downloadDSV(activeDoc, req, res, {...options, delimiter: '💩'});
     }));
 
     this._app.get('/api/docs/:docId/download/xlsx', canView, withDoc(async (activeDoc, req, res) => {

app/server/lib/DocPluginManager.ts

@@ -146,6 +146,8 @@ export class DocPluginManager {
         '.xlsx' : 'Excel',
         '.json' : 'JSON',
         '.csv' : 'CSV',
+        '.tsv' : 'TSV',
+        '.dsv' : 'PSV',
       };
       const fileType = extToType[path.extname(fileName)] || path.extname(fileName);
       throw new Error(`Failed to parse ${fileType} file.\nError: ${messages.join("; ")}`);

app/server/lib/ExportDSV.ts

@@ -11,26 +11,38 @@ import * as express from 'express';
 // promisify csv
 bluebird.promisifyAll(csv);
 
+export interface DownloadDsvOptions extends DownloadOptions {
+  delimiter: Delimiter;
+}
+
+type Delimiter = ',' | '\t' | '💩';
+
 /**
- * Converts `activeDoc` to a CSV and sends the converted data through `res`.
+ * Converts `activeDoc` to delimiter-separated values (e.g. CSV) and sends
+ * the converted data through `res`.
  */
-export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
-                                  res: express.Response, options: DownloadOptions) {
-  log.info('Generating .csv file...');
-  const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, header} = options;
+export async function downloadDSV(
+  activeDoc: ActiveDoc,
+  req: express.Request,
+  res: express.Response,
+  options: DownloadDsvOptions
+) {
+  const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, delimiter, header} = options;
+  const extension = getDSVFileExtension(delimiter);
+  log.info(`Generating ${extension} file...`);
   const data = viewSectionId ?
-    await makeCSVFromViewSection({
+    await makeDSVFromViewSection({
       activeDoc, viewSectionId, sortOrder: sortOrder || null, filters: filters || null,
-      linkingFilter: linkingFilter || null, header, req
+      linkingFilter: linkingFilter || null, header, delimiter, req
     }) :
-    await makeCSVFromTable({activeDoc, tableId, header, req});
-  res.set('Content-Type', 'text/csv');
-  res.setHeader('Content-Disposition', contentDisposition(filename + '.csv'));
+    await makeDSVFromTable({activeDoc, tableId, header, delimiter, req});
+  res.set('Content-Type', getDSVMimeType(delimiter));
+  res.setHeader('Content-Disposition', contentDisposition(filename + extension));
   res.send(data);
 }
 
 /**
- * Returns a csv stream of a view section that can be transformed or parsed.
+ * Returns a DSV stream of a view section that can be transformed or parsed.
  *
  * See https://github.com/wdavidw/node-csv for API details.
  *
@@ -40,13 +52,21 @@ export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request,
  * @param {Integer[]} options.activeSortOrder (optional) - overriding sort order.
  * @param {Filter[]} options.filters (optional) - filters defined from ui.
  * @param {FilterColValues} options.linkingFilter (optional) - linking filter defined from ui.
+ * @param {Delimiter} options.delimiter - delimiter to separate fields with
  * @param {string} options.header (optional) - which field of the column to use as header
  * @param {express.Request} options.req - the request object.
  *
- * @return {Promise<string>} Promise for the resulting CSV.
+ * @return {Promise<string>} Promise for the resulting DSV.
  */
-export async function makeCSVFromViewSection({
-  activeDoc, viewSectionId, sortOrder = null, filters = null, linkingFilter = null, header, req
+export async function makeDSVFromViewSection({
+  activeDoc,
+  viewSectionId,
+  sortOrder = null,
+  filters = null,
+  linkingFilter = null,
+  delimiter,
+  header,
+  req
 }: {
   activeDoc: ActiveDoc,
   viewSectionId: number,
@@ -54,28 +74,31 @@ export async function makeCSVFromViewSection({
   filters: Filter[] | null,
   linkingFilter: FilterColValues | null,
   header?: ExportHeader,
+  delimiter: Delimiter,
   req: express.Request
 }) {
 
   const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, linkingFilter, req);
-  const file = convertToCsv(data, { header });
+  const file = convertToDsv(data, { header, delimiter });
   return file;
 }
 
 /**
- * Returns a csv stream of a table that can be transformed or parsed.
+ * Returns a DSV stream of a table that can be transformed or parsed.
  *
  * @param {Object} options - options for the export.
  * @param {Object} options.activeDoc - the activeDoc that the table being converted belongs to.
  * @param {Integer} options.tableId - id of the table to export.
+ * @param {Delimiter} options.delimiter  - delimiter to separate fields with
  * @param {string} options.header (optional) - which field of the column to use as header
  * @param {express.Request} options.req - the request object.
  *
- * @return {Promise<string>} Promise for the resulting CSV.
+ * @return {Promise<string>} Promise for the resulting DSV.
  */
-export async function makeCSVFromTable({ activeDoc, tableId, header, req }: {
+export async function makeDSVFromTable({ activeDoc, tableId, delimiter, header, req }: {
   activeDoc: ActiveDoc,
   tableId: string,
+  delimiter: Delimiter,
   header?: ExportHeader,
   req: express.Request
 }) {
@@ -93,24 +116,63 @@ export async function makeCSVFromTable({ activeDoc, tableId, header, req }: {
   }
 
   const data = await exportTable(activeDoc, tableRef, req);
-  const file = convertToCsv(data, { header });
+  const file = convertToDsv(data, { header, delimiter });
   return file;
 }
 
-function convertToCsv({
-  rowIds,
-  access,
-  columns: viewColumns,
-}: ExportData, options: { header?: ExportHeader }) {
+interface ConvertToDsvOptions {
+  delimiter: Delimiter;
+  header?: ExportHeader;
+}
 
+function convertToDsv(data: ExportData, options: ConvertToDsvOptions) {
+  const {rowIds, access, columns: viewColumns} = data;
+  const {delimiter, header} = options;
   // create formatters for columns
   const formatters = viewColumns.map(col => col.formatter);
   // Arrange the data into a row-indexed matrix, starting with column headers.
-  const colPropertyAsHeader = options.header ?? 'label';
+  const colPropertyAsHeader = header ?? 'label';
   const csvMatrix = [viewColumns.map(col => col[colPropertyAsHeader])];
   // populate all the rows with values as strings
   rowIds.forEach(row => {
     csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter(row))));
   });
-  return csv.stringifyAsync(csvMatrix);
+  return csv.stringifyAsync(csvMatrix, {delimiter});
+}
+
+type DSVFileExtension = '.csv' | '.tsv' | '.dsv';
+
+function getDSVFileExtension(delimiter: Delimiter): DSVFileExtension {
+  switch (delimiter) {
+    case ',': {
+      return '.csv';
+    }
+    case '\t': {
+      return '.tsv';
+    }
+    case '💩': {
+      return '.dsv';
+    }
+  }
+}
+
+type DSVMimeType =
+  | 'text/csv'
+  // Reference: https://www.iana.org/assignments/media-types/text/tab-separated-values
+  | 'text/tab-separated-values'
+  // Note: not a registered MIME type, hence the "x-" prefix.
+  | 'text/x-doo-separated-values';
+
+function getDSVMimeType(delimiter: Delimiter): DSVMimeType {
+  switch (delimiter) {
+    case ',': {
+      return 'text/csv';
+    }
+    case '\t': {
+      return 'text/tab-separated-values';
+    }
+    case '💩': {
+      return 'text/x-doo-separated-values';
+    }
+  }
 }

plugins/core/manifest.yml

@@ -5,7 +5,7 @@ components:
   safePython: sandbox/main.py
 contributions:
   fileParsers:
-    - fileExtensions: ["csv", "tsv", "txt"]
+    - fileExtensions: ["csv", "tsv", "dsv", "txt"]
       parseFile:
         component: safePython
         name: csv_parser

test/fixtures/export-dsv/text.dsv

@@ -0,0 +1,4 @@
+Foo💩Bar💩Id is Baz Label is this💩Link💩Formula
+1💩a💩hello💩grist https://www.getgrist.com/💩a --- grist https://www.getgrist.com/
+2💩b ,d💩world💩https://www.getgrist.com/💩b ,d --- https://www.getgrist.com/
+3💩"the ""quote marks"" ?"💩💩💩"the ""quote marks"" ? --- "

test/fixtures/export-tsv/text.tsv

@@ -0,0 +1,4 @@
+Foo	Bar	Id is Baz Label is this	Link	Formula
+1	a	hello	grist https://www.getgrist.com/	a --- grist https://www.getgrist.com/
+2	b ,d	world	https://www.getgrist.com/	b ,d --- https://www.getgrist.com/
+3	"the ""quote marks"" ?"			"the ""quote marks"" ? --- "

test/nbrowser/Export.ntest.js

@@ -1,4 +1,4 @@
-import { assert } from 'mocha-webdriver';
+import { assert, driver } from 'mocha-webdriver';
 import { $, gu, test } from 'test/nbrowser/gristUtil-nbrowser';
 
 const fse = require('fs-extra');
@@ -35,7 +35,10 @@ describe('Export.ntest', function() {
     await $('.test-tb-share').click();
     // Once the menu opens, get the href of the link.
     await $('.grist-floating-menu').wait();
-    const href = await $('.grist-floating-menu a:contains(CSV)').wait().getAttribute('href');
+    const submenu = $('.test-tb-share-option:contains(Export as...)');
+    await driver.withActions(a => a.move({origin: submenu.elem()}));
+    const href = await $('.grist-floating-menu a:contains(Comma Separated Values)').wait()
+      .getAttribute('href');
     // Download the data at the link and compare to expected.
     const resp = await axios.get(href, {responseType: 'text', headers});
     assert.equal(resp.headers['content-disposition'],
@@ -50,7 +53,10 @@ describe('Export.ntest', function() {
     await $('.test-tb-share').click();
     // Once the menu opens, get the href of the link.
     await $('.grist-floating-menu').wait();
-    const href = await $('.grist-floating-menu a:contains(CSV)').wait().getAttribute('href');
+    const submenu = $('.test-tb-share-option:contains(Export as...)');
+    await driver.withActions(a => a.move({origin: submenu.elem()}));
+    const href = await $('.grist-floating-menu a:contains(Comma Separated Values)').wait()
+      .getAttribute('href');
     // Download the data at the link and compare to expected.
     const resp = await axios.get(href, {responseType: 'text', headers});
     assert.equal(resp.data, dataExpected.sorted);