(core) Custom Widget column mapping feature.

Summary:
Exposing new API in CustomSectionAPI for column mapping.

The custom widget can call configure method (or use a ready method) with additional parameter "columns".
This parameter is a list of column names that should be mapped by the user.
Mapping configuration is exposed through an additional method in the CustomSectionAPI "mappings". It is also available
through the onRecord(s) event.

This DIFF is connected with PR for grist-widgets repository https://github.com/gristlabs/grist-widget/pull/15

Design document and discussion: https://grist.quip.com/Y2waA8h8Zuzu/Custom-Widget-field-mapping

Test Plan: browser tests

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3241

app/plugin/CustomSectionAPI-ti.ts

@@ -4,22 +4,41 @@
 import * as t from "ts-interface-checker";
 // tslint:disable:object-literal-key-quotes
 
-export const RequestedInteractionOptions = t.iface([], {
+export const ColumnToMap = t.iface([], {
+  "name": "string",
+  "title": t.opt(t.union("string", "null")),
+  "type": t.opt("string"),
+  "optional": t.opt("boolean"),
+  "allowMultiple": t.opt("boolean"),
+});
+
+export const ColumnsToMap = t.array(t.union("string", "ColumnToMap"));
+
+export const InteractionOptionsRequest = t.iface([], {
   "requiredAccess": t.opt("string"),
   "hasCustomOptions": t.opt("boolean"),
+  "columns": t.opt("ColumnsToMap"),
 });
 
 export const InteractionOptions = t.iface([], {
   "accessLevel": "string",
 });
 
+export const WidgetColumnMap = t.iface([], {
+  [t.indexKey]: t.union("string", t.array("string"), "null"),
+});
+
 export const CustomSectionAPI = t.iface([], {
-  "configure": t.func("void", t.param("customOptions", "RequestedInteractionOptions")),
+  "configure": t.func("void", t.param("customOptions", "InteractionOptionsRequest")),
+  "mappings": t.func(t.union("WidgetColumnMap", "null")),
 });
 
 const exportedTypeSuite: t.ITypeSuite = {
-  RequestedInteractionOptions,
+  ColumnToMap,
+  ColumnsToMap,
+  InteractionOptionsRequest,
   InteractionOptions,
+  WidgetColumnMap,
   CustomSectionAPI,
 };
 export default exportedTypeSuite;

app/plugin/CustomSectionAPI.ts

@@ -2,6 +2,31 @@
  * API definitions for CustomSection plugins.
  */
 
+export interface ColumnToMap {
+  /**
+   * Column name that Widget expects. Must be a valid JSON property name.
+   */
+  name: string;
+  /**
+   * Title or short description of a column (used as a label in section mapping).
+   */
+  title?: string|null,
+  /**
+   * Column type, by default ANY.
+   */
+  type?: string, // GristType, TODO: ts-interface-checker doesn't know how to parse this
+  /**
+   * Mark column as optional all columns are required by default.
+   */
+  optional?: boolean
+  /**
+   * Allow multiple column assignment, the result will be list of mapped table column names.
+   */
+  allowMultiple?: boolean,
+}
+
+export type ColumnsToMap = (string|ColumnToMap)[];
+
 /**
  * Initial message sent by the CustomWidget with initial requirements.
  */
@@ -16,18 +41,37 @@ export interface InteractionOptionsRequest {
    * can use to show custom options screen.
    */
   hasCustomOptions?: boolean,
+  /**
+   * Tells Grist what columns Custom Widget expects and allows user to map between existing column names
+   * and those requested by Custom Widget.
+   */
+  columns?: ColumnsToMap,
 }
 
 /**
  * Widget configuration set and approved by Grist, sent as part of ready message.
  */
-export interface InteractionOptions {
+export interface InteractionOptions{
   /**
    * Granted access level.
    */
-  accessLevel: string
+   accessLevel: string,
+}
+
+/**
+ * Current columns mapping between viewFields in section and Custom widget.
+ */
+export interface WidgetColumnMap {
+  [key: string]: string|string[]|null
 }
 
 export interface CustomSectionAPI {
+  /**
+   * Initial request from a Custom Widget that wants to declare its requirements.
+   */
   configure(customOptions: InteractionOptionsRequest): Promise<void>;
+  /**
+   * Returns current widget configuration (if requested through configuration method).
+   */
+  mappings(): Promise<WidgetColumnMap|null>;
 }

app/plugin/GristData-ti.ts

@@ -27,9 +27,12 @@ export const RowRecord = t.iface([], {
   [t.indexKey]: "CellValue",
 });
 
+export const GristType = t.union(t.lit('Any'), t.lit('Attachments'), t.lit('Blob'), t.lit('Bool'), t.lit('Choice'), t.lit('ChoiceList'), t.lit('Date'), t.lit('DateTime'), t.lit('Id'), t.lit('Int'), t.lit('ManualSortPos'), t.lit('Numeric'), t.lit('PositionNumber'), t.lit('Ref'), t.lit('RefList'), t.lit('Text'));
+
 const exportedTypeSuite: t.ITypeSuite = {
   GristObjCode,
   CellValue,
   RowRecord,
+  GristType,
 };
 export default exportedTypeSuite;

app/plugin/GristData.ts

@@ -21,3 +21,7 @@ export interface RowRecord {
   id: number;
   [colId: string]: CellValue;
 }
+
+export type GristType = 'Any' | 'Attachments' | 'Blob' | 'Bool' | 'Choice' | 'ChoiceList' |
+  'Date' | 'DateTime' |
+  'Id' | 'Int' | 'ManualSortPos' | 'Numeric' | 'PositionNumber' | 'Ref' | 'RefList' | 'Text';

app/plugin/grist-plugin-api.ts

@@ -18,7 +18,8 @@
 
 // tslint:disable:no-console
 
-import { CustomSectionAPI, InteractionOptions } from './CustomSectionAPI';
+import { ColumnsToMap, CustomSectionAPI, InteractionOptions, InteractionOptionsRequest,
+         WidgetColumnMap } from './CustomSectionAPI';
 import { GristAPI, GristDocAPI, GristView, RPC_GRISTAPI_INTERFACE } from './GristAPI';
 import { RowRecord } from './GristData';
 import { ImportSource, ImportSourceAPI, InternalImportSourceAPI } from './InternalImportSourceAPI';
@@ -70,6 +71,97 @@ export const docApi: GristDocAPI & GristView = {
 
 export const on = rpc.on.bind(rpc);
 
+// For custom widgets that support custom columns mappings store current configuration
+// in a memory.
+
+// Actual cached value. Undefined means that widget hasn't asked for configuration yet.
+// Here we are storing serialized configuration instead of actual one, since widget can
+// mutate returned value.
+let _mappingsCache: WidgetColumnMap|null|undefined;
+// Since widget needs to ask for mappings during onRecord and onRecords event, we will reuse
+// current request if available;
+let _activeRefreshReq: Promise<void>|null = null;
+// Remember columns requested during ready call.
+let _columnsToMap: ColumnsToMap|undefined;
+
+async function getMappingsIfChanged(data: any): Promise<WidgetColumnMap|null> {
+  const uninitialized = _mappingsCache === undefined;
+  if (data.mappingsChange || uninitialized) {
+    // If no active request.
+    if (!_activeRefreshReq) {
+      // Request for new mappings.
+      _activeRefreshReq = sectionApi
+        .mappings()
+        // Store it in global variable.
+        .then(mappings => void (_mappingsCache = mappings))
+        // Clear current request variable.
+        .finally(() => _activeRefreshReq = null);
+    }
+    await _activeRefreshReq;
+  }
+  return _mappingsCache ? JSON.parse(JSON.stringify(_mappingsCache)) : null;
+}
+
+/**
+ * Renames columns in the result using columns mapping configuration passed in ready method.
+ * Returns null if not all required columns were mapped or not widget doesn't support
+ * custom column mapping.
+ */
+export function mapColumnNames(data: any, options = {
+  columns: _columnsToMap,
+  mappings: _mappingsCache
+}) {
+  // If not column configuration was requested or
+  // table has no rows, return original data.
+  if (!options.columns) {
+    return data;
+  }
+  // If we haven't received columns configuration return null.
+  if (!options.mappings) {
+    return null;
+  }
+  // If we are renaming names for whole table, but it is empty, don't do anything.
+  if (Array.isArray(data) && data.length === 0) {
+    return data;
+  }
+
+  // Prepare convert function - a function that will take record returned from Grist
+  // and convert it to a new record with mapped field names;
+  // Convert function will consists of several transformations:
+  const transformations: ((from: any, to: any) => void)[] = [];
+  // First transformation is for copying id field:
+  transformations.push((from, to) => to.id = from.id);
+  // Helper function to test if a column was configured as optional.
+  function isOptional(col: string) {
+    return Boolean(
+      // Columns passed as strings are required.
+      !options.columns?.includes(col)
+      && options.columns?.find(c => typeof c === 'object' && c?.name === col && c.optional)
+    );
+  }
+  // For each widget column in mapping.
+  for(const widgetCol in options.mappings) {
+    // Get column from Grist.
+    const gristCol = options.mappings[widgetCol];
+    // Copy column as series (multiple values)
+    if (Array.isArray(gristCol) && gristCol.length) {
+      transformations.push((from, to) => {
+        to[widgetCol] = gristCol.map(col => from[col]);
+      });
+      // Copy column directly under widget column name.
+    } else if (!Array.isArray(gristCol) && gristCol) {
+      transformations.push((from, to) => to[widgetCol] = from[gristCol]);
+    } else if (!isOptional(widgetCol)) {
+      // Column was not configured but was required.
+      return null;
+    }
+  }
+  // Finally assemble function to convert a single record.
+  const convert = (rec: any) => transformations.reduce((obj, tran) => { tran(rec, obj); return obj; }, {} as any);
+  // Transform all records (or a single one depending on the arguments).
+  return Array.isArray(data) ? data.map(convert) : convert(data);
+}
+
 // For custom widgets, add a handler that will be called whenever the
 // row with the cursor changes - either by switching to a different row, or
 // by some value within the row potentially changing.  Handler may
@@ -77,17 +169,16 @@ export const on = rpc.on.bind(rpc);
 // any row.
 // TODO: currently this will be called even if the content of a different row
 // changes.
-export function onRecord(callback: (data: RowRecord | null) => unknown) {
+export function onRecord(callback: (data: RowRecord | null, mappings: WidgetColumnMap | null) => unknown) {
   on('message', async function(msg) {
     if (!msg.tableId || !msg.rowId) { return; }
     const rec = await docApi.fetchSelectedRecord(msg.rowId);
-    callback(rec);
+    callback(rec, await getMappingsIfChanged(msg));
   });
 }
-
 // For custom widgets, add a handler that will be called whenever the
 // selected records change.  Handler will be called with a list of records.
-export function onRecords(callback: (data: RowRecord[]) => unknown) {
+export function onRecords(callback: (data: RowRecord[], mappings: WidgetColumnMap | null) => unknown) {
   on('message', async function(msg) {
     if (!msg.tableId || !msg.dataChange) { return; }
     const data = await docApi.fetchSelectedTable();
@@ -100,7 +191,7 @@ export function onRecords(callback: (data: RowRecord[]) => unknown) {
       }
       rows.push(row);
     }
-    callback(rows);
+    callback(rows, await getMappingsIfChanged(msg));
   });
 }
 
@@ -146,14 +237,17 @@ export async function addImporter(name: string, path: string, mode: 'fullscreen'
   });
 }
 
+interface ReadyPayload extends Omit<InteractionOptionsRequest, "hasCustomOptions"> {
+  /**
+   * Handler that will be called by Grist to open additional configuration panel inside the Custom Widget.
+   */
+  onEditOptions: () => unknown;
+}
 /**
  * Declare that a component is prepared to receive messages from the outside world.
  * Grist will not attempt to communicate with it until this method is called.
  */
-export function ready(settings?: {
-  requiredAccess?: string,
-  onEditOptions: () => unknown
-}): void {
+export function ready(settings?: ReadyPayload): void {
   if (settings && settings.onEditOptions) {
     rpc.registerFunc('editOptions', settings.onEditOptions);
   }
@@ -161,10 +255,13 @@ export function ready(settings?: {
   void (async function() {
     await rpc.sendReadyMessage();
     if (settings) {
-      await sectionApi.configure({
-          requiredAccess : settings.requiredAccess,
-          hasCustomOptions: Boolean(settings.onEditOptions)
-      }).catch((err: unknown) => console.error(err));
+      const options = {
+        ...(settings),
+        hasCustomOptions: Boolean(settings.onEditOptions),
+      };
+      delete options.onEditOptions;
+      _columnsToMap = options.columns;
+      await sectionApi.configure(options).catch((err: unknown) => console.error(err));
     }
   })();
 }