(core) Improve object serialization, to help get RECORD data to Custom Widgets.

Summary:
- Change RECORD's dates_as_str default to False.
- Reimplement objtype encode_object/decode_object with less machinery.
- Implement encoding of dicts (with string keys).
- Make lists and dicts encode values recursively.
- Implement encoding/decoding in the client
- Decode automatically in plugins' fetchSelectedTable/Record, with an option to skip.

Test Plan: Tested manually, not sure what tests may be affected yet.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2593

app/common/ValueFormatter.ts

@@ -4,35 +4,41 @@ import {CellValue} from 'app/common/DocActions';
 import * as gristTypes from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
 import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
+import {decodeObject, GristDateTime} from 'app/plugin/objtypes';
+import isPlainObject = require('lodash/isPlainObject');
 import * as moment from 'moment-timezone';
 
-// Some text to show on cells whose values are pending.
-export const PENDING_DATA_PLACEHOLDER = "Loading...";
+export {PENDING_DATA_PLACEHOLDER} from 'app/plugin/objtypes';
 
 /**
- * Formats a custom object received as a value in a DocAction, as "Constructor(args...)".
- * E.g. ["Foo", 1, 2, 3] becomes the string "Foo(1, 2, 3)".
+ * Formats a value of any type generically (with no type-specific options).
  */
-export function formatObject(args: [string, ...any[]]): string {
-  const objType = args[0], objArgs = args.slice(1);
-  switch (objType) {
-    case 'L': return JSON.stringify(objArgs);
-    // First arg is seconds since epoch (moment takes ms), second arg is timezone
-    case 'D': return moment.tz(objArgs[0] * 1000, objArgs[1]).format("YYYY-MM-DD HH:mm:ssZ");
-    case 'd': return moment.tz(objArgs[0] * 1000, 'UTC').format("YYYY-MM-DD");
-    case 'R': return `${objArgs[0]}[${objArgs[1]}]`;
-    case 'E': return gristTypes.formatError(args);
-    case 'U': return String(args[1]);
-    case 'P': return PENDING_DATA_PLACEHOLDER;
-  }
-  return objType + "(" + JSON.stringify(objArgs).slice(1, -1) + ")";
+export function formatUnknown(value: CellValue): string {
+  return formatHelper(decodeObject(value));
 }
 
 /**
- * Formats a value of unknown type, using formatObject() for encoded objects.
+ * Formats a decoded Grist value for displaying it. For top-level values, formats them the way we
+ * like to see them in a cell or in, say, CSV export. For lists and objects, nested values are
+ * formatted slighly differently, with quoted strings and ISO format for dates.
  */
-export function formatUnknown(value: any): string {
-  return gristTypes.isObject(value) ? formatObject(value) : (value == null ? "" : String(value));
+function formatHelper(value: unknown, isTopLevel: boolean = true): string {
+  if (typeof value === 'object' && value) {
+    if (Array.isArray(value)) {
+      return '[' + value.map(v => formatHelper(v, false)).join(', ') + ']';
+    } else if (isPlainObject(value)) {
+      const obj: any = value;
+      const items = Object.keys(obj).map(k => `${JSON.stringify(k)}: ${formatHelper(obj[k], false)}`);
+      return '{' + items.join(', ') + '}';
+    } else if (isTopLevel && value instanceof GristDateTime) {
+      return moment(value).tz(value.timezone).format("YYYY-MM-DD HH:mm:ssZ");
+    }
+    return String(value);
+  }
+  if (isTopLevel) {
+    return (value == null ? "" : String(value));
+  }
+  return JSON.stringify(value);
 }
 
 export type IsRightTypeFunc = (value: CellValue) => boolean;

app/common/gristTypes.ts

@@ -6,9 +6,16 @@ import isString = require('lodash/isString');
 export type GristType = 'Any' | 'Attachments' | 'Blob' | 'Bool' | 'Choice' | 'Date' | 'DateTime' |
   'Id' | 'Int' | 'ManualSortPos' | 'Numeric' | 'PositionNumber' | 'Ref' | 'RefList' | 'Text';
 
+export type GristTypeInfo =
+  {type: 'DateTime', timezone: string} |
+  {type: 'Ref', tableId: string} |
+  {type: Exclude<GristType, 'DateTime'|'Ref'>};
+
+
 // Letter codes for CellValue types encoded as [code, args...] tuples.
 export const enum GristObjCode {
   List            = 'L',
+  Dict            = 'O',
   DateTime        = 'D',
   Date            = 'd',
   Reference       = 'R',
@@ -51,6 +58,36 @@ export function getDefaultForType(colType: string, options: {sqlFormatted?: bool
   return (_defaultValues[type as GristType] || _defaultValues.Any)[options.sqlFormatted ? 1 : 0];
 }
 
+/**
+ * Convert a type like 'Numeric', 'DateTime:America/New_York', or 'Ref:Table1' to a GristTypeInfo
+ * object.
+ */
+export function extractInfoFromColType(colType: string): GristTypeInfo {
+  const colon = colType.indexOf(':');
+  const [type, arg] = (colon === -1) ? [colType] : [colType.slice(0, colon), colType.slice(colon + 1)];
+  return (type === 'Ref') ? {type, tableId: String(arg)} :
+    (type === 'DateTime') ? {type, timezone: String(arg)} :
+    {type} as GristTypeInfo;
+}
+
+/**
+ * Re-encodes a CellValue of a given Grist type as a value suitable to use in an Any column. E.g.
+ *    reencodeAsAny(123, 'Numeric') -> 123
+ *    reencodeAsAny(123, 'Date') -> ['d', 123]
+ *    reencodeAsAny(123, 'Reference', 'Table1') -> ['R', 'Table1', 123]
+ */
+export function reencodeAsAny(value: CellValue, typeInfo: GristTypeInfo): CellValue {
+  if (typeof value === 'number') {
+    switch (typeInfo.type) {
+      case 'Date': return ['d', value];
+      case 'DateTime': return ['D', value, typeInfo.timezone];
+      case 'Ref': return ['R', typeInfo.tableId, value];
+    }
+  }
+  return value;
+}
+
+
 /**
  * Returns whether a value (as received in a DocAction) represents a custom object.
  */
@@ -88,20 +125,6 @@ export function isEmptyList(value: CellValue): boolean {
   return Array.isArray(value) && value.length === 1 && value[0] === GristObjCode.List;
 }
 
-/**
- * Formats a raised exception (a value for which isRaisedException is true) for display in a cell.
- * This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!"
- */
-export function formatError(value: [string, ...any[]]): string {
-  const errName = value[1];
-  switch (errName) {
-    case 'ZeroDivisionError': return '#DIV/0!';
-    case 'UnmarshallableError': return value[3] || ('#' + errName);
-    case 'InvalidTypedValue': return `#Invalid ${value[2]}: ${value[3]}`;
-  }
-  return '#' + errName;
-}
-
 function isNumber(v: CellValue) { return typeof v === 'number' || typeof v === 'boolean'; }
 function isNumberOrNull(v: CellValue) { return isNumber(v) || v === null; }
 function isBoolean(v: CellValue) { return typeof v === 'boolean' || v === 1 || v === 0; }

app/plugin/GristData.ts

@@ -1,4 +1,4 @@
-export type CellValue = number|string|boolean|null|[string, any?];
+export type CellValue = number|string|boolean|null|[string, ...unknown[]];
 
 export interface RowRecord {
   id: number;

app/plugin/grist-plugin-api.ts

@@ -21,6 +21,7 @@
 import { GristAPI, GristDocAPI, GristView, RPC_GRISTAPI_INTERFACE } from './GristAPI';
 import { RowRecord } from './GristData';
 import { ImportSource, ImportSourceAPI, InternalImportSourceAPI } from './InternalImportSourceAPI';
+import { decodeObject, mapValues } from './objtypes';
 import { RenderOptions, RenderTarget } from './RenderOptions';
 import { checkers } from './TypeCheckers';
 
@@ -39,9 +40,26 @@ export const rpc: Rpc = new Rpc({logger: createRpcLogger()});
 export const api = rpc.getStub<GristAPI>(RPC_GRISTAPI_INTERFACE, checkers.GristAPI);
 export const coreDocApi = rpc.getStub<GristDocAPI>('GristDocAPI@grist', checkers.GristDocAPI);
 export const viewApi = rpc.getStub<GristView>('GristView', checkers.GristView);
-export const docApi = {
+
+export const docApi: GristDocAPI & GristView = {
   ...coreDocApi,
   ...viewApi,
+
+  // Change fetchSelectedTable() to decode data by default, replacing e.g. ['D', timestamp] with
+  // a moment date. New option `keepEncoded` skips the decoding step.
+  async fetchSelectedTable(options: {keepEncoded?: boolean} = {}) {
+    const table = await viewApi.fetchSelectedTable();
+    return options.keepEncoded ? table :
+      mapValues<any[], any[]>(table, (col) => col.map(decodeObject));
+  },
+
+  // Change fetchSelectedRecord() to decode data by default, replacing e.g. ['D', timestamp] with
+  // a moment date. New option `keepEncoded` skips the decoding step.
+  async fetchSelectedRecord(rowId: number, options: {keepEncoded?: boolean} = {}) {
+    const rec = await viewApi.fetchSelectedRecord(rowId);
+    return options.keepEncoded ? rec :
+      mapValues(rec, decodeObject);
+  }
 };
 
 export const on = rpc.on.bind(rpc);

app/plugin/objtypes.ts

@@ -0,0 +1,189 @@
+/**
+ * Encodes and decodes Grist encoding of values, mirroring similar Python functions in
+ * sandbox/grist/objtypes.py.
+ */
+// tslint:disable:max-classes-per-file
+
+import {CellValue} from 'app/plugin/GristData';
+import isPlainObject = require('lodash/isPlainObject');
+
+// The text to show on cells whose values are pending.
+export const PENDING_DATA_PLACEHOLDER = "Loading...";
+
+/**
+ * A GristDate is just a JS Date object whose toString() method returns YYYY-MM-DD.
+ */
+export class GristDate extends Date {
+  public static fromGristValue(epochSec: number): GristDate {
+    return new GristDate(epochSec * 1000);
+  }
+
+  public toString() {
+    return this.toISOString().slice(0, 10);
+  }
+}
+
+/**
+ * A GristDateTime is a JS Date with an added timezone field. Its toString() returns the date in
+ * ISO format. To create a timezone-aware momentjs object, use:
+ *
+ *    moment(d).tz(d.timezone)
+ */
+export class GristDateTime extends Date {
+  public static fromGristValue(epochSec: number, timezone: string): GristDateTime {
+    return Object.assign(new GristDateTime(epochSec * 1000), {timezone});
+  }
+
+  public timezone: string;
+  public toString() { return this.toISOString(); }
+}
+
+/**
+ * A Reference represents a reference to a row in a table. It is simply a pair of a string tableId
+ * and a numeric rowId.
+ */
+export class Reference {
+  constructor(public tableId: string, public rowId: number) {}
+
+  public toString(): string {
+    return `${this.tableId}[${this.rowId}]`;
+  }
+}
+
+/**
+ * A RaisedException represents a formula error. It includes the exception name, message, and
+ * optional details.
+ */
+export class RaisedException {
+  constructor(public name: string, public message?: string, public details?: string) {}
+
+  /**
+   * This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!"
+   */
+  public toString() {
+    switch (this.name) {
+      case 'ZeroDivisionError': return '#DIV/0!';
+      case 'UnmarshallableError': return this.details || ('#' + this.name);
+      case 'InvalidTypedValue': return `#Invalid ${this.message}: ${this.details}`;
+    }
+    return '#' + this.name;
+  }
+}
+
+/**
+ * An UnknownValue is a fallback for values that we don't handle otherwise, e.g. of a Python
+ * formula returned a function object, or a value we fail to decode.
+ * It is typically the Python repr() string of the value.
+ */
+export class UnknownValue {
+  // When encoding an unknown value, get a best-effort string form of it.
+  public static safeRepr(value: unknown): string {
+    try {
+      return String(value);
+    } catch (e) {
+      return `<${typeof value}>`;
+    }
+  }
+
+  constructor(public value: unknown) {}
+  public toString() {
+    return String(this.value);
+  }
+}
+
+/**
+ * A trivial placeholder for a value that's not yet available.
+ */
+export class PendingValue {
+  public toString() {
+    return PENDING_DATA_PLACEHOLDER;
+  }
+}
+
+/**
+ * Produces a Grist-encoded version of the value, e.g. turning a Date into ['d', timestamp].
+ * Returns ['U', repr(value)] if it fails to encode otherwise.
+ *
+ * TODO Add tests. This is not yet used for anything.
+ */
+export function encodeObject(value: unknown): CellValue {
+  try {
+    switch (typeof value) {
+      case 'string':
+      case 'number':
+      case 'boolean':
+        return value;
+    }
+    if (value == null) {
+      return null;
+    } else if (value instanceof Reference) {
+      return ['R', value.tableId, value.rowId];
+    } else if (value instanceof Date) {
+      const timestamp = value.valueOf() / 1000;
+      if ('timezone' in value) {
+        return ['D', timestamp, (value as GristDateTime).timezone];
+      } else {
+        // TODO Depending on how it's used, may want to return ['d', timestamp] for UTC midnight.
+        return ['D', timestamp, 'UTC'];
+      }
+    } else if (value instanceof RaisedException) {
+      return ['E', value.name, value.message, value.details];
+    } else if (Array.isArray(value)) {
+      return ['L', ...value.map(encodeObject)];
+    } else if (isPlainObject(value)) {
+      return ['O', mapValues(value as any, encodeObject, {sort: true})];
+    }
+  } catch (e) {
+    // Fall through to return a best-effort representation.
+  }
+  // We either don't know how to convert the value, or failed during the conversion. Instead we
+  // return an "UnmarshallableValue" object, with repr() of the value to show to the user.
+  return ['U', UnknownValue.safeRepr(value)];
+}
+
+
+/**
+ * Given a Grist-encoded value, returns an object represented by it.
+ * If the type code is unknown, or construction fails for any reason, returns an UnknownValue.
+ */
+export function decodeObject(value: CellValue): unknown {
+  if (!Array.isArray(value)) {
+    return value;
+  }
+  const code: string = value[0];
+  const args: any[] = value.slice(1);
+  let err: Error|undefined;
+  try {
+    switch (code) {
+      case 'D': return GristDateTime.fromGristValue(args[0], String(args[1]));
+      case 'd': return GristDate.fromGristValue(args[0]);
+      case 'E': return new RaisedException(args[0], args[1], args[2]);
+      case 'L': return (args as CellValue[]).map(decodeObject);
+      case 'O': return mapValues(args[0] as {[key: string]: CellValue}, decodeObject, {sort: true});
+      case 'P': return new PendingValue();
+      case 'R': return new Reference(String(args[0]), args[1]);
+      case 'U': return new UnknownValue(args[0]);
+    }
+  } catch (e) {
+    err = e;
+  }
+  // If we can't decode, return an UnknownValue with some attempt to represent what we couldn't
+  // decode as long as some info about the error if any.
+  return new UnknownValue(`${code}(${JSON.stringify(args).slice(1, -1)})` +
+    (err ? `#${err.name}(${err.message})` : ''));
+}
+
+// Like lodash's mapValues, with support for sorting keys, for friendlier output.
+export function mapValues<A, B>(
+  sourceObj: {[key: string]: A}, mapper: (value: A) => B, options: {sort?: boolean} = {}
+): {[key: string]: B} {
+  const result: {[key: string]: B} = {};
+  const keys = Object.keys(sourceObj);
+  if (options.sort) {
+    keys.sort();
+  }
+  for (const key of keys) {
+    result[key] = mapper(sourceObj[key]);
+  }
+  return result;
+}