(core) Improve object serialization, to help get RECORD data to Custom Widgets.

Summary:
- Change RECORD's dates_as_str default to False.
- Reimplement objtype encode_object/decode_object with less machinery.
- Implement encoding of dicts (with string keys).
- Make lists and dicts encode values recursively.
- Implement encoding/decoding in the client
- Decode automatically in plugins' fetchSelectedTable/Record, with an option to skip.

Test Plan: Tested manually, not sure what tests may be affected yet.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2593

app/common/ValueFormatter.ts

@@ -4,35 +4,41 @@ import {CellValue} from 'app/common/DocActions';
 import * as gristTypes from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
 import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
+import {decodeObject, GristDateTime} from 'app/plugin/objtypes';
+import isPlainObject = require('lodash/isPlainObject');
 import * as moment from 'moment-timezone';
 
-// Some text to show on cells whose values are pending.
-export const PENDING_DATA_PLACEHOLDER = "Loading...";
+export {PENDING_DATA_PLACEHOLDER} from 'app/plugin/objtypes';
 
 /**
- * Formats a custom object received as a value in a DocAction, as "Constructor(args...)".
- * E.g. ["Foo", 1, 2, 3] becomes the string "Foo(1, 2, 3)".
+ * Formats a value of any type generically (with no type-specific options).
  */
-export function formatObject(args: [string, ...any[]]): string {
-  const objType = args[0], objArgs = args.slice(1);
-  switch (objType) {
-    case 'L': return JSON.stringify(objArgs);
-    // First arg is seconds since epoch (moment takes ms), second arg is timezone
-    case 'D': return moment.tz(objArgs[0] * 1000, objArgs[1]).format("YYYY-MM-DD HH:mm:ssZ");
-    case 'd': return moment.tz(objArgs[0] * 1000, 'UTC').format("YYYY-MM-DD");
-    case 'R': return `${objArgs[0]}[${objArgs[1]}]`;
-    case 'E': return gristTypes.formatError(args);
-    case 'U': return String(args[1]);
-    case 'P': return PENDING_DATA_PLACEHOLDER;
-  }
-  return objType + "(" + JSON.stringify(objArgs).slice(1, -1) + ")";
+export function formatUnknown(value: CellValue): string {
+  return formatHelper(decodeObject(value));
 }
 
 /**
- * Formats a value of unknown type, using formatObject() for encoded objects.
+ * Formats a decoded Grist value for displaying it. For top-level values, formats them the way we
+ * like to see them in a cell or in, say, CSV export. For lists and objects, nested values are
+ * formatted slighly differently, with quoted strings and ISO format for dates.
  */
-export function formatUnknown(value: any): string {
-  return gristTypes.isObject(value) ? formatObject(value) : (value == null ? "" : String(value));
+function formatHelper(value: unknown, isTopLevel: boolean = true): string {
+  if (typeof value === 'object' && value) {
+    if (Array.isArray(value)) {
+      return '[' + value.map(v => formatHelper(v, false)).join(', ') + ']';
+    } else if (isPlainObject(value)) {
+      const obj: any = value;
+      const items = Object.keys(obj).map(k => `${JSON.stringify(k)}: ${formatHelper(obj[k], false)}`);
+      return '{' + items.join(', ') + '}';
+    } else if (isTopLevel && value instanceof GristDateTime) {
+      return moment(value).tz(value.timezone).format("YYYY-MM-DD HH:mm:ssZ");
+    }
+    return String(value);
+  }
+  if (isTopLevel) {
+    return (value == null ? "" : String(value));
+  }
+  return JSON.stringify(value);
 }
 
 export type IsRightTypeFunc = (value: CellValue) => boolean;

app/common/gristTypes.ts

@@ -6,9 +6,16 @@ import isString = require('lodash/isString');
 export type GristType = 'Any' | 'Attachments' | 'Blob' | 'Bool' | 'Choice' | 'Date' | 'DateTime' |
   'Id' | 'Int' | 'ManualSortPos' | 'Numeric' | 'PositionNumber' | 'Ref' | 'RefList' | 'Text';
 
+export type GristTypeInfo =
+  {type: 'DateTime', timezone: string} |
+  {type: 'Ref', tableId: string} |
+  {type: Exclude<GristType, 'DateTime'|'Ref'>};
+
+
 // Letter codes for CellValue types encoded as [code, args...] tuples.
 export const enum GristObjCode {
   List            = 'L',
+  Dict            = 'O',
   DateTime        = 'D',
   Date            = 'd',
   Reference       = 'R',
@@ -51,6 +58,36 @@ export function getDefaultForType(colType: string, options: {sqlFormatted?: bool
   return (_defaultValues[type as GristType] || _defaultValues.Any)[options.sqlFormatted ? 1 : 0];
 }
 
+/**
+ * Convert a type like 'Numeric', 'DateTime:America/New_York', or 'Ref:Table1' to a GristTypeInfo
+ * object.
+ */
+export function extractInfoFromColType(colType: string): GristTypeInfo {
+  const colon = colType.indexOf(':');
+  const [type, arg] = (colon === -1) ? [colType] : [colType.slice(0, colon), colType.slice(colon + 1)];
+  return (type === 'Ref') ? {type, tableId: String(arg)} :
+    (type === 'DateTime') ? {type, timezone: String(arg)} :
+    {type} as GristTypeInfo;
+}
+
+/**
+ * Re-encodes a CellValue of a given Grist type as a value suitable to use in an Any column. E.g.
+ *    reencodeAsAny(123, 'Numeric') -> 123
+ *    reencodeAsAny(123, 'Date') -> ['d', 123]
+ *    reencodeAsAny(123, 'Reference', 'Table1') -> ['R', 'Table1', 123]
+ */
+export function reencodeAsAny(value: CellValue, typeInfo: GristTypeInfo): CellValue {
+  if (typeof value === 'number') {
+    switch (typeInfo.type) {
+      case 'Date': return ['d', value];
+      case 'DateTime': return ['D', value, typeInfo.timezone];
+      case 'Ref': return ['R', typeInfo.tableId, value];
+    }
+  }
+  return value;
+}
+
+
 /**
  * Returns whether a value (as received in a DocAction) represents a custom object.
  */
@@ -88,20 +125,6 @@ export function isEmptyList(value: CellValue): boolean {
   return Array.isArray(value) && value.length === 1 && value[0] === GristObjCode.List;
 }
 
-/**
- * Formats a raised exception (a value for which isRaisedException is true) for display in a cell.
- * This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!"
- */
-export function formatError(value: [string, ...any[]]): string {
-  const errName = value[1];
-  switch (errName) {
-    case 'ZeroDivisionError': return '#DIV/0!';
-    case 'UnmarshallableError': return value[3] || ('#' + errName);
-    case 'InvalidTypedValue': return `#Invalid ${value[2]}: ${value[3]}`;
-  }
-  return '#' + errName;
-}
-
 function isNumber(v: CellValue) { return typeof v === 'number' || typeof v === 'boolean'; }
 function isNumberOrNull(v: CellValue) { return isNumber(v) || v === null; }
 function isBoolean(v: CellValue) { return typeof v === 'boolean' || v === 1 || v === 0; }