From c54e910fd6999715610643ecb97e558f508267c6 Mon Sep 17 00:00:00 2001 From: Louis Delbosc Date: Thu, 16 Mar 2023 22:37:24 +0100 Subject: [PATCH] Export table schema (#459) * add endpoint * Add table-schema transformation data --- app/client/ui/ViewLayoutMenu.ts | 2 +- app/common/UserAPI.ts | 8 +- app/common/WidgetOptions.ts | 11 ++ app/server/lib/DocApi.ts | 21 ++++ app/server/lib/Export.ts | 4 +- app/server/lib/ExportTableSchema.ts | 151 ++++++++++++++++++++++++++++ test/server/lib/DocApi.ts | 56 +++++++++++ 7 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 app/common/WidgetOptions.ts create mode 100644 app/server/lib/ExportTableSchema.ts diff --git a/app/client/ui/ViewLayoutMenu.ts b/app/client/ui/ViewLayoutMenu.ts index 49b9e297..67a60ec4 100644 --- a/app/client/ui/ViewLayoutMenu.ts +++ b/app/client/ui/ViewLayoutMenu.ts @@ -79,7 +79,7 @@ export function makeViewLayoutMenu(viewSection: ViewSectionRec, isReadonly: bool menuItemLink({ href: gristDoc.getCsvLink(), target: '_blank', download: ''}, t("Download as CSV"), testId('download-section')), menuItemLink({ href: gristDoc.getXlsxActiveViewLink(), target: '_blank', download: ''}, - t("Download as XLSX"), testId('download-section')), + t("Download as XLSX"), testId('download-section')), dom.maybe((use) => ['detail', 'single'].includes(use(viewSection.parentKey)), () => menuItemCmd(allCommands.editLayout, t("Edit Card Layout"), dom.cls('disabled', isReadonly))), diff --git a/app/common/UserAPI.ts b/app/common/UserAPI.ts index 8379e59a..53a28c83 100644 --- a/app/common/UserAPI.ts +++ b/app/common/UserAPI.ts @@ -370,7 +370,7 @@ export interface UserAPI { } /** - * Parameters for the download CSV and XLSX endpoint (/download/csv & /download/csv). + * Parameters for the download CSV and XLSX endpoint (/download/table-schema & /download/csv & /download/csv). */ export interface DownloadDocParams { tableId: string; @@ -418,6 +418,7 @@ export interface DocAPI { getDownloadUrl(template?: boolean): string; getDownloadXlsxUrl(params?: DownloadDocParams): string; getDownloadCsvUrl(params: DownloadDocParams): string; + getDownloadTableSchemaUrl(params: DownloadDocParams): string; /** * Exports current document to the Google Drive as a spreadsheet file. To invoke this method, first * acquire "code" via Google Auth Endpoint (see ShareMenu.ts for an example). @@ -920,6 +921,11 @@ export class DocAPIImpl extends BaseAPI implements DocAPI { return this._url + '/download/csv?' + encodeQueryParams({...params}); } + public getDownloadTableSchemaUrl(params: DownloadDocParams) { + // We spread `params` to work around TypeScript being overly cautious. + return this._url + '/download/table-schema?' + encodeQueryParams({...params}); + } + public async sendToDrive(code: string, title: string): Promise<{url: string}> { const url = new URL(`${this._url}/send-to-drive`); url.searchParams.append('title', title); diff --git a/app/common/WidgetOptions.ts b/app/common/WidgetOptions.ts new file mode 100644 index 00000000..c51a62c8 --- /dev/null +++ b/app/common/WidgetOptions.ts @@ -0,0 +1,11 @@ +import {NumberFormatOptions} from 'app/common/NumberFormat'; + +export interface WidgetOptions extends NumberFormatOptions { + textColor?: 'string'; + fillColor?: 'string'; + alignment?: 'left' | 'center' | 'right'; + dateFormat?: string; + timeFormat?: string; + widget?: 'HyperLink'; + choices?: Array; +} \ No newline at end of file diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index 544c78cb..e25f5bca 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -36,6 +36,7 @@ import {IDocWorkerMap} from "app/server/lib/DocWorkerMap"; import {DownloadOptions, parseExportParameters} from "app/server/lib/Export"; import {downloadCSV} from "app/server/lib/ExportCSV"; import {downloadXLSX} from "app/server/lib/ExportXLSX"; +import {collectTableSchemaInFrictionlessFormat} from "app/server/lib/ExportTableSchema"; import {expressWrap} from 'app/server/lib/expressWrap'; import {filterDocumentInPlace} from "app/server/lib/filterUtils"; import {googleAuthTokenMiddleware} from "app/server/lib/GoogleAuth"; @@ -871,6 +872,26 @@ export class DocWorkerApi { res.json(result); })); + this._app.get('/api/docs/:docId/download/table-schema', canView, withDoc(async (activeDoc, req, res) => { + const doc = await this._dbManager.getDoc(req); + const options = this._getDownloadOptions(req, doc.name); + const tableSchema = await collectTableSchemaInFrictionlessFormat(activeDoc, req, options); + const apiPath = await this._grist.getResourceUrl(doc, 'api'); + const query = new URLSearchParams(req.query as {[key: string]: string}); + const tableSchemaPath = `${apiPath}/download/csv?${query.toString()}`; + res.send({ + format: "csv", + mediatype: "text/csv", + encoding: "utf-8", + path: tableSchemaPath, + dialect: { + delimiter: ",", + doubleQuote: true, + }, + ...tableSchema, + }); + })); + this._app.get('/api/docs/:docId/download/csv', canView, withDoc(async (activeDoc, req, res) => { // Query DB for doc metadata to get the doc title. const {name: docTitle} = await this._dbManager.getDoc(req); diff --git a/app/server/lib/Export.ts b/app/server/lib/Export.ts index 26749002..cd0cdde3 100644 --- a/app/server/lib/Export.ts +++ b/app/server/lib/Export.ts @@ -204,7 +204,8 @@ export async function exportTable( label: tc.label, type: displayCol.type, widgetOptions, - parentPos: tc.parentPos + parentPos: tc.parentPos, + description: displayCol.description, }; }).filter(tc => tc !== emptyCol); @@ -279,6 +280,7 @@ export async function exportSection( label: col.label, type: col.type, parentPos: col.parentPos, + description: col.description, widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions), }; }; diff --git a/app/server/lib/ExportTableSchema.ts b/app/server/lib/ExportTableSchema.ts new file mode 100644 index 00000000..bd467db4 --- /dev/null +++ b/app/server/lib/ExportTableSchema.ts @@ -0,0 +1,151 @@ +import * as express from 'express'; +import {ApiError} from 'app/common/ApiError'; +import {WidgetOptions} from 'app/common/WidgetOptions'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {DownloadOptions, exportTable} from 'app/server/lib/Export'; + +interface ExportColumn { + id: number; + colId: string; + label: string; + type: string; + widgetOptions: WidgetOptions; + description?: string; + parentPos: number; +} + +interface FrictionlessFormat { + name: string; + title: string; + schema: { + fields: { + name: string; + type: string; + description?: string; + format?: string; + bareNumber?: boolean; + groupChar?: string; + decimalChar?: string; + gristFormat?: string; + constraint?: {}; + trueValue?: string[]; + falseValue?: string[]; + }[] + } +} + +/** + * Return a table schema for frictionless interoperability + * + * See https://specs.frictionlessdata.io/table-schema/#page-frontmatter-title for spec + * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. + * @param {Object} options - options to get the table ID + * @return {Promise} Promise for the resulting schema. + */ +export async function collectTableSchemaInFrictionlessFormat( + activeDoc: ActiveDoc, + req: express.Request, + options: DownloadOptions +): Promise { + const {tableId} = options; + if (!activeDoc.docData) { + throw new Error('No docData in active document'); + } + + // Look up the table to make a CSV from. + const settings = activeDoc.docData.docSettings(); + const tables = activeDoc.docData.getMetaTable('_grist_Tables'); + const tableRef = tables.findRow('tableId', tableId); + + if (tableRef === 0) { + throw new ApiError(`Table ${tableId} not found.`, 404); + } + + const data = await exportTable(activeDoc, tableRef, req); + const tableSchema = columnsToTableSchema(tableId, data, settings.locale); + return tableSchema; +} + +function columnsToTableSchema( + tableId: string, + {tableName, columns}: {tableName: string, columns: ExportColumn[]}, + locale: string, +): FrictionlessFormat { + return { + name: tableId.toLowerCase().replace(/_/g, '-'), + title: tableName, + schema: { + fields: columns.map(col => ({ + name: col.label, + ...(col.description ? {description: col.description} : {}), + ...buildTypeField(col, locale), + })), + } + }; +} + +function buildTypeField(col: ExportColumn, locale: string) { + const type = col.type.split(':', 1)[0]; + switch (type) { + case 'Text': + return { + type: 'string', + format: col.widgetOptions.widget === 'HyperLink' ? 'uri' : 'default', + }; + case 'Numeric': + return { + type: 'number', + bareNumber: col.widgetOptions?.numMode === 'decimal', + ...getNumberSeparators(locale), + }; + case 'Integer': + return { + type: 'integer', + bareNumber: col.widgetOptions?.numMode === 'decimal', + groupChar: getNumberSeparators(locale).groupChar, + }; + case 'Date': + return { + type: 'date', + format: 'any', + gristFormat: col.widgetOptions?.dateFormat || 'YYYY-MM-DD', + }; + case 'DateTime': + return { + type: 'datetime', + format: 'any', + gristFormat: `${col.widgetOptions?.dateFormat} ${col.widgetOptions?.timeFormat}`, + }; + case 'Bool': + return { + type: 'boolean', + trueValue: ['TRUE'], + falseValue: ['FALSE'], + }; + case 'Choice': + return { + type: 'string', + constraints: {enum: col.widgetOptions?.choices}, + }; + case 'ChoiceList': + return { + type: 'array', + constraints: {enum: col.widgetOptions?.choices}, + }; + case 'Reference': + return {type: 'string'}; + case 'ReferenceList': + return {type: 'array'}; + default: + return {type: 'string'}; + } +} + +function getNumberSeparators(locale: string) { + const numberWithGroupAndDecimalSeparator = 1000.1; + const parts = Intl.NumberFormat(locale).formatToParts(numberWithGroupAndDecimalSeparator); + return { + groupChar: parts.find(obj => obj.type === 'group')?.value, + decimalChar: parts.find(obj => obj.type === 'decimal')?.value, + }; +} \ No newline at end of file diff --git a/test/server/lib/DocApi.ts b/test/server/lib/DocApi.ts index 796701c1..a705deeb 100644 --- a/test/server/lib/DocApi.ts +++ b/test/server/lib/DocApi.ts @@ -2138,6 +2138,62 @@ function testDocApi() { assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' }); }); + it("GET /docs/{did}/download/table-schema serves table-schema-encoded document", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema?tableId=Foo`, chimpy); + assert.equal(resp.status, 200); + const expected = { + format: "csv", + mediatype: "text/csv", + encoding: "utf-8", + dialect: { + delimiter: ",", + doubleQuote: true, + }, + name: 'foo', + title: 'Foo', + schema: { + fields: [{ + name: 'A', + type: 'string', + format: 'default', + }, { + name: 'B', + type: 'string', + format: 'default', + }] + } + }; + assert.deepInclude(resp.data, expected); + + const resp2 = await axios.get(resp.data.path, chimpy); + assert.equal(resp2.status, 200); + assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n'); + }); + + it("GET /docs/{did}/download/table-schema respects permissions", async function() { + // kiwi has no access to TestDoc + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema?tableId=Table1`, kiwi); + assert.equal(resp.status, 403); + assert.deepEqual(resp.data, {"error":"No view access"}); + }); + + it("GET /docs/{did}/download/table-schema returns 404 if tableId is invalid", async function() { + const resp = await axios.get( + `${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema?tableId=MissingTableId`, + chimpy, + ); + assert.equal(resp.status, 404); + assert.deepEqual(resp.data, { error: 'Table MissingTableId not found.' }); + }); + + it("GET /docs/{did}/download/table-schema returns 400 if tableId is missing", async function() { + const resp = await axios.get( + `${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema`, chimpy); + assert.equal(resp.status, 400); + assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' }); + }); + + it("GET /docs/{did}/download/xlsx serves XLSX-encoded document", async function() { const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/xlsx?tableId=Table1`, chimpy); assert.equal(resp.status, 200);