Export table schema (#459)

* add endpoint
* Add table-schema transformation data

app/common/UserAPI.ts

@@ -370,7 +370,7 @@ export interface UserAPI {
 }
 
 /**
- * Parameters for the download CSV and XLSX endpoint (/download/csv & /download/csv).
+ * Parameters for the download CSV and XLSX endpoint (/download/table-schema & /download/csv & /download/csv).
  */
  export interface DownloadDocParams {
   tableId: string;
@@ -418,6 +418,7 @@ export interface DocAPI {
   getDownloadUrl(template?: boolean): string;
   getDownloadXlsxUrl(params?: DownloadDocParams): string;
   getDownloadCsvUrl(params: DownloadDocParams): string;
+  getDownloadTableSchemaUrl(params: DownloadDocParams): string;
   /**
    * Exports current document to the Google Drive as a spreadsheet file. To invoke this method, first
    * acquire "code" via Google Auth Endpoint (see ShareMenu.ts for an example).
@@ -920,6 +921,11 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
     return this._url + '/download/csv?' + encodeQueryParams({...params});
   }
 
+  public getDownloadTableSchemaUrl(params: DownloadDocParams) {
+    // We spread `params` to work around TypeScript being overly cautious.
+    return this._url + '/download/table-schema?' + encodeQueryParams({...params});
+  }
+
   public async sendToDrive(code: string, title: string): Promise<{url: string}> {
     const url = new URL(`${this._url}/send-to-drive`);
     url.searchParams.append('title', title);

app/common/WidgetOptions.ts

@@ -0,0 +1,11 @@
+import {NumberFormatOptions} from 'app/common/NumberFormat';
+
+export interface WidgetOptions extends NumberFormatOptions {
+  textColor?: 'string';
+  fillColor?: 'string';
+  alignment?: 'left' | 'center' | 'right';
+  dateFormat?: string;
+  timeFormat?: string;
+  widget?: 'HyperLink';
+  choices?: Array<string>;
+}

app/server/lib/DocApi.ts

@@ -36,6 +36,7 @@ import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
 import {DownloadOptions, parseExportParameters} from "app/server/lib/Export";
 import {downloadCSV} from "app/server/lib/ExportCSV";
 import {downloadXLSX} from "app/server/lib/ExportXLSX";
+import {collectTableSchemaInFrictionlessFormat} from "app/server/lib/ExportTableSchema";
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {filterDocumentInPlace} from "app/server/lib/filterUtils";
 import {googleAuthTokenMiddleware} from "app/server/lib/GoogleAuth";
@@ -871,6 +872,26 @@ export class DocWorkerApi {
       res.json(result);
     }));
 
+    this._app.get('/api/docs/:docId/download/table-schema', canView, withDoc(async (activeDoc, req, res) => {
+      const doc = await this._dbManager.getDoc(req);
+      const options = this._getDownloadOptions(req, doc.name);
+      const tableSchema = await collectTableSchemaInFrictionlessFormat(activeDoc, req, options);
+      const apiPath = await this._grist.getResourceUrl(doc, 'api');
+      const query = new URLSearchParams(req.query as {[key: string]: string});
+      const tableSchemaPath = `${apiPath}/download/csv?${query.toString()}`;
+      res.send({
+        format: "csv",
+        mediatype: "text/csv",
+        encoding: "utf-8",
+        path: tableSchemaPath,
+        dialect: {
+          delimiter: ",",
+          doubleQuote: true,
+        },
+        ...tableSchema,
+      });
+    }));
+
     this._app.get('/api/docs/:docId/download/csv', canView, withDoc(async (activeDoc, req, res) => {
       // Query DB for doc metadata to get the doc title.
       const {name: docTitle} = await this._dbManager.getDoc(req);

app/server/lib/Export.ts

@@ -204,7 +204,8 @@ export async function exportTable(
       label: tc.label,
       type: displayCol.type,
       widgetOptions,
-      parentPos: tc.parentPos
+      parentPos: tc.parentPos,
+      description: displayCol.description,
     };
   }).filter(tc => tc !== emptyCol);
 
@@ -279,6 +280,7 @@ export async function exportSection(
       label: col.label,
       type: col.type,
       parentPos: col.parentPos,
+      description: col.description,
       widgetOptions: Object.assign(colWidgetOptions, fieldWidgetOptions),
     };
   };

app/server/lib/ExportTableSchema.ts

@@ -0,0 +1,151 @@
+import * as express from 'express';
+import {ApiError} from 'app/common/ApiError';
+import {WidgetOptions} from 'app/common/WidgetOptions';
+import {ActiveDoc} from 'app/server/lib/ActiveDoc';
+import {DownloadOptions, exportTable} from 'app/server/lib/Export';
+
+interface ExportColumn {
+  id: number;
+  colId: string;
+  label: string;
+  type: string;
+  widgetOptions: WidgetOptions;
+  description?: string;
+  parentPos: number;
+}
+
+interface FrictionlessFormat {
+  name: string;
+  title: string;
+  schema: {
+    fields: {
+      name: string;
+      type: string;
+      description?: string;
+      format?: string;
+      bareNumber?: boolean;
+      groupChar?: string;
+      decimalChar?: string;
+      gristFormat?: string;
+      constraint?: {};
+      trueValue?: string[];
+      falseValue?: string[];
+    }[]
+  }
+}
+
+/**
+ * Return a table schema for frictionless interoperability
+ *
+ * See https://specs.frictionlessdata.io/table-schema/#page-frontmatter-title for spec
+ * @param {Object} activeDoc - the activeDoc that the table being converted belongs to.
+ * @param {Object} options - options to get the table ID
+ * @return {Promise<FrictionlessFormat>} Promise for the resulting schema.
+ */
+export async function collectTableSchemaInFrictionlessFormat(
+  activeDoc: ActiveDoc,
+  req: express.Request,
+  options: DownloadOptions
+): Promise<FrictionlessFormat> {
+  const {tableId} = options;
+  if (!activeDoc.docData) {
+    throw new Error('No docData in active document');
+  }
+
+  // Look up the table to make a CSV from.
+  const settings = activeDoc.docData.docSettings();
+  const tables = activeDoc.docData.getMetaTable('_grist_Tables');
+  const tableRef = tables.findRow('tableId', tableId);
+
+  if (tableRef === 0) {
+    throw new ApiError(`Table ${tableId} not found.`, 404);
+  }
+
+  const data = await exportTable(activeDoc, tableRef, req);
+  const tableSchema = columnsToTableSchema(tableId, data, settings.locale);
+  return tableSchema;
+}
+
+function columnsToTableSchema(
+  tableId: string,
+  {tableName, columns}: {tableName: string, columns: ExportColumn[]},
+  locale: string,
+): FrictionlessFormat {
+  return {
+    name: tableId.toLowerCase().replace(/_/g, '-'),
+    title: tableName,
+    schema: {
+      fields: columns.map(col => ({
+        name: col.label,
+        ...(col.description ? {description: col.description} : {}),
+        ...buildTypeField(col, locale),
+      })),
+    }
+  };
+}
+
+function buildTypeField(col: ExportColumn, locale: string) {
+  const type = col.type.split(':', 1)[0];
+  switch (type) {
+    case 'Text':
+      return {
+        type: 'string',
+        format: col.widgetOptions.widget === 'HyperLink' ? 'uri' : 'default',
+      };
+    case 'Numeric':
+      return {
+        type: 'number',
+        bareNumber: col.widgetOptions?.numMode === 'decimal',
+        ...getNumberSeparators(locale),
+      };
+    case 'Integer':
+      return {
+        type: 'integer',
+        bareNumber: col.widgetOptions?.numMode === 'decimal',
+        groupChar: getNumberSeparators(locale).groupChar,
+      };
+    case 'Date':
+      return {
+        type: 'date',
+        format: 'any',
+        gristFormat: col.widgetOptions?.dateFormat || 'YYYY-MM-DD',
+      };
+    case 'DateTime':
+      return {
+        type: 'datetime',
+        format: 'any',
+        gristFormat: `${col.widgetOptions?.dateFormat} ${col.widgetOptions?.timeFormat}`,
+      };
+    case 'Bool':
+      return {
+        type: 'boolean',
+        trueValue: ['TRUE'],
+        falseValue: ['FALSE'],
+      };
+    case 'Choice':
+      return {
+        type: 'string',
+        constraints: {enum: col.widgetOptions?.choices},
+      };
+    case 'ChoiceList':
+      return {
+        type: 'array',
+        constraints: {enum: col.widgetOptions?.choices},
+      };
+    case 'Reference':
+      return {type: 'string'};
+    case 'ReferenceList':
+      return {type: 'array'};
+    default:
+      return {type: 'string'};
+  }
+}
+
+function getNumberSeparators(locale: string) {
+  const numberWithGroupAndDecimalSeparator = 1000.1;
+  const parts = Intl.NumberFormat(locale).formatToParts(numberWithGroupAndDecimalSeparator);
+  return {
+    groupChar: parts.find(obj => obj.type === 'group')?.value,
+    decimalChar: parts.find(obj => obj.type === 'decimal')?.value,
+  };
+}

test/server/lib/DocApi.ts

@@ -2138,6 +2138,62 @@ function testDocApi() {
     assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' });
   });
 
+  it("GET /docs/{did}/download/table-schema serves table-schema-encoded document", async function() {
+    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema?tableId=Foo`, chimpy);
+    assert.equal(resp.status, 200);
+    const expected = {
+      format: "csv",
+      mediatype: "text/csv",
+      encoding: "utf-8",
+      dialect: {
+        delimiter: ",",
+        doubleQuote: true,
+      },
+      name: 'foo',
+      title: 'Foo',
+      schema: {
+        fields: [{
+          name: 'A',
+          type: 'string',
+          format: 'default',
+        }, {
+          name: 'B',
+          type: 'string',
+          format: 'default',
+        }]
+      }
+    };
+    assert.deepInclude(resp.data, expected);
+
+    const resp2 = await axios.get(resp.data.path, chimpy);
+    assert.equal(resp2.status, 200);
+    assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n');
+  });
+
+  it("GET /docs/{did}/download/table-schema respects permissions", async function() {
+    // kiwi has no access to TestDoc
+    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema?tableId=Table1`, kiwi);
+    assert.equal(resp.status, 403);
+    assert.deepEqual(resp.data, {"error":"No view access"});
+  });
+
+  it("GET /docs/{did}/download/table-schema returns 404 if tableId is invalid", async function() {
+    const resp = await axios.get(
+      `${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema?tableId=MissingTableId`,
+      chimpy,
+    );
+    assert.equal(resp.status, 404);
+    assert.deepEqual(resp.data, { error: 'Table MissingTableId not found.' });
+  });
+
+  it("GET /docs/{did}/download/table-schema returns 400 if tableId is missing", async function() {
+    const resp = await axios.get(
+      `${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema`, chimpy);
+    assert.equal(resp.status, 400);
+    assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' });
+  });
+
+
   it("GET /docs/{did}/download/xlsx serves XLSX-encoded document", async function() {
     const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/xlsx?tableId=Table1`, chimpy);
     assert.equal(resp.status, 200);