(core) Improve object serialization, to help get RECORD data to Custom Widgets.

Summary:
- Change RECORD's dates_as_str default to False.
- Reimplement objtype encode_object/decode_object with less machinery.
- Implement encoding of dicts (with string keys).
- Make lists and dicts encode values recursively.
- Implement encoding/decoding in the client
- Decode automatically in plugins' fetchSelectedTable/Record, with an option to skip.

Test Plan: Tested manually, not sure what tests may be affected yet.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2593

app/common/ValueFormatter.ts

@@ -4,35 +4,41 @@ import {CellValue} from 'app/common/DocActions';
 import * as gristTypes from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
 import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
+import {decodeObject, GristDateTime} from 'app/plugin/objtypes';
+import isPlainObject = require('lodash/isPlainObject');
 import * as moment from 'moment-timezone';
 
-// Some text to show on cells whose values are pending.
+export {PENDING_DATA_PLACEHOLDER} from 'app/plugin/objtypes';
-export const PENDING_DATA_PLACEHOLDER = "Loading...";
 
 /**
- * Formats a custom object received as a value in a DocAction, as "Constructor(args...)".
+ * Formats a value of any type generically (with no type-specific options).
- * E.g. ["Foo", 1, 2, 3] becomes the string "Foo(1, 2, 3)".
  */
-export function formatObject(args: [string, ...any[]]): string {
+export function formatUnknown(value: CellValue): string {
-  const objType = args[0], objArgs = args.slice(1);
+  return formatHelper(decodeObject(value));
-  switch (objType) {
-    case 'L': return JSON.stringify(objArgs);
-    // First arg is seconds since epoch (moment takes ms), second arg is timezone
-    case 'D': return moment.tz(objArgs[0] * 1000, objArgs[1]).format("YYYY-MM-DD HH:mm:ssZ");
-    case 'd': return moment.tz(objArgs[0] * 1000, 'UTC').format("YYYY-MM-DD");
-    case 'R': return `${objArgs[0]}[${objArgs[1]}]`;
-    case 'E': return gristTypes.formatError(args);
-    case 'U': return String(args[1]);
-    case 'P': return PENDING_DATA_PLACEHOLDER;
-  }
-  return objType + "(" + JSON.stringify(objArgs).slice(1, -1) + ")";
 }
 
 /**
- * Formats a value of unknown type, using formatObject() for encoded objects.
+ * Formats a decoded Grist value for displaying it. For top-level values, formats them the way we
+ * like to see them in a cell or in, say, CSV export. For lists and objects, nested values are
+ * formatted slighly differently, with quoted strings and ISO format for dates.
  */
-export function formatUnknown(value: any): string {
+function formatHelper(value: unknown, isTopLevel: boolean = true): string {
-  return gristTypes.isObject(value) ? formatObject(value) : (value == null ? "" : String(value));
+  if (typeof value === 'object' && value) {
+    if (Array.isArray(value)) {
+      return '[' + value.map(v => formatHelper(v, false)).join(', ') + ']';
+    } else if (isPlainObject(value)) {
+      const obj: any = value;
+      const items = Object.keys(obj).map(k => `${JSON.stringify(k)}: ${formatHelper(obj[k], false)}`);
+      return '{' + items.join(', ') + '}';
+    } else if (isTopLevel && value instanceof GristDateTime) {
+      return moment(value).tz(value.timezone).format("YYYY-MM-DD HH:mm:ssZ");
+    }
+    return String(value);
+  }
+  if (isTopLevel) {
+    return (value == null ? "" : String(value));
+  }
+  return JSON.stringify(value);
 }
 
 export type IsRightTypeFunc = (value: CellValue) => boolean;

app/common/gristTypes.ts

@@ -6,9 +6,16 @@ import isString = require('lodash/isString');
 export type GristType = 'Any' | 'Attachments' | 'Blob' | 'Bool' | 'Choice' | 'Date' | 'DateTime' |
   'Id' | 'Int' | 'ManualSortPos' | 'Numeric' | 'PositionNumber' | 'Ref' | 'RefList' | 'Text';
 
+export type GristTypeInfo =
+  {type: 'DateTime', timezone: string} |
+  {type: 'Ref', tableId: string} |
+  {type: Exclude<GristType, 'DateTime'|'Ref'>};
+
+
 // Letter codes for CellValue types encoded as [code, args...] tuples.
 export const enum GristObjCode {
   List            = 'L',
+  Dict            = 'O',
   DateTime        = 'D',
   Date            = 'd',
   Reference       = 'R',
@@ -51,6 +58,36 @@ export function getDefaultForType(colType: string, options: {sqlFormatted?: bool
   return (_defaultValues[type as GristType] || _defaultValues.Any)[options.sqlFormatted ? 1 : 0];
 }
 
+/**
+ * Convert a type like 'Numeric', 'DateTime:America/New_York', or 'Ref:Table1' to a GristTypeInfo
+ * object.
+ */
+export function extractInfoFromColType(colType: string): GristTypeInfo {
+  const colon = colType.indexOf(':');
+  const [type, arg] = (colon === -1) ? [colType] : [colType.slice(0, colon), colType.slice(colon + 1)];
+  return (type === 'Ref') ? {type, tableId: String(arg)} :
+    (type === 'DateTime') ? {type, timezone: String(arg)} :
+    {type} as GristTypeInfo;
+}
+
+/**
+ * Re-encodes a CellValue of a given Grist type as a value suitable to use in an Any column. E.g.
+ *    reencodeAsAny(123, 'Numeric') -> 123
+ *    reencodeAsAny(123, 'Date') -> ['d', 123]
+ *    reencodeAsAny(123, 'Reference', 'Table1') -> ['R', 'Table1', 123]
+ */
+export function reencodeAsAny(value: CellValue, typeInfo: GristTypeInfo): CellValue {
+  if (typeof value === 'number') {
+    switch (typeInfo.type) {
+      case 'Date': return ['d', value];
+      case 'DateTime': return ['D', value, typeInfo.timezone];
+      case 'Ref': return ['R', typeInfo.tableId, value];
+    }
+  }
+  return value;
+}
+
+
 /**
  * Returns whether a value (as received in a DocAction) represents a custom object.
  */
@@ -88,20 +125,6 @@ export function isEmptyList(value: CellValue): boolean {
   return Array.isArray(value) && value.length === 1 && value[0] === GristObjCode.List;
 }
 
-/**
- * Formats a raised exception (a value for which isRaisedException is true) for display in a cell.
- * This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!"
- */
-export function formatError(value: [string, ...any[]]): string {
-  const errName = value[1];
-  switch (errName) {
-    case 'ZeroDivisionError': return '#DIV/0!';
-    case 'UnmarshallableError': return value[3] || ('#' + errName);
-    case 'InvalidTypedValue': return `#Invalid ${value[2]}: ${value[3]}`;
-  }
-  return '#' + errName;
-}
-
 function isNumber(v: CellValue) { return typeof v === 'number' || typeof v === 'boolean'; }
 function isNumberOrNull(v: CellValue) { return isNumber(v) || v === null; }
 function isBoolean(v: CellValue) { return typeof v === 'boolean' || v === 1 || v === 0; }

app/plugin/GristData.ts

@@ -1,4 +1,4 @@
-export type CellValue = number|string|boolean|null|[string, any?];
+export type CellValue = number|string|boolean|null|[string, ...unknown[]];
 
 export interface RowRecord {
   id: number;

app/plugin/grist-plugin-api.ts

@@ -21,6 +21,7 @@
 import { GristAPI, GristDocAPI, GristView, RPC_GRISTAPI_INTERFACE } from './GristAPI';
 import { RowRecord } from './GristData';
 import { ImportSource, ImportSourceAPI, InternalImportSourceAPI } from './InternalImportSourceAPI';
+import { decodeObject, mapValues } from './objtypes';
 import { RenderOptions, RenderTarget } from './RenderOptions';
 import { checkers } from './TypeCheckers';
 
@@ -39,9 +40,26 @@ export const rpc: Rpc = new Rpc({logger: createRpcLogger()});
 export const api = rpc.getStub<GristAPI>(RPC_GRISTAPI_INTERFACE, checkers.GristAPI);
 export const coreDocApi = rpc.getStub<GristDocAPI>('GristDocAPI@grist', checkers.GristDocAPI);
 export const viewApi = rpc.getStub<GristView>('GristView', checkers.GristView);
-export const docApi = {
+
+export const docApi: GristDocAPI & GristView = {
   ...coreDocApi,
   ...viewApi,
+
+  // Change fetchSelectedTable() to decode data by default, replacing e.g. ['D', timestamp] with
+  // a moment date. New option `keepEncoded` skips the decoding step.
+  async fetchSelectedTable(options: {keepEncoded?: boolean} = {}) {
+    const table = await viewApi.fetchSelectedTable();
+    return options.keepEncoded ? table :
+      mapValues<any[], any[]>(table, (col) => col.map(decodeObject));
+  },
+
+  // Change fetchSelectedRecord() to decode data by default, replacing e.g. ['D', timestamp] with
+  // a moment date. New option `keepEncoded` skips the decoding step.
+  async fetchSelectedRecord(rowId: number, options: {keepEncoded?: boolean} = {}) {
+    const rec = await viewApi.fetchSelectedRecord(rowId);
+    return options.keepEncoded ? rec :
+      mapValues(rec, decodeObject);
+  }
 };
 
 export const on = rpc.on.bind(rpc);

app/plugin/objtypes.ts

@@ -0,0 +1,189 @@
+/**
+ * Encodes and decodes Grist encoding of values, mirroring similar Python functions in
+ * sandbox/grist/objtypes.py.
+ */
+// tslint:disable:max-classes-per-file
+
+import {CellValue} from 'app/plugin/GristData';
+import isPlainObject = require('lodash/isPlainObject');
+
+// The text to show on cells whose values are pending.
+export const PENDING_DATA_PLACEHOLDER = "Loading...";
+
+/**
+ * A GristDate is just a JS Date object whose toString() method returns YYYY-MM-DD.
+ */
+export class GristDate extends Date {
+  public static fromGristValue(epochSec: number): GristDate {
+    return new GristDate(epochSec * 1000);
+  }
+
+  public toString() {
+    return this.toISOString().slice(0, 10);
+  }
+}
+
+/**
+ * A GristDateTime is a JS Date with an added timezone field. Its toString() returns the date in
+ * ISO format. To create a timezone-aware momentjs object, use:
+ *
+ *    moment(d).tz(d.timezone)
+ */
+export class GristDateTime extends Date {
+  public static fromGristValue(epochSec: number, timezone: string): GristDateTime {
+    return Object.assign(new GristDateTime(epochSec * 1000), {timezone});
+  }
+
+  public timezone: string;
+  public toString() { return this.toISOString(); }
+}
+
+/**
+ * A Reference represents a reference to a row in a table. It is simply a pair of a string tableId
+ * and a numeric rowId.
+ */
+export class Reference {
+  constructor(public tableId: string, public rowId: number) {}
+
+  public toString(): string {
+    return `${this.tableId}[${this.rowId}]`;
+  }
+}
+
+/**
+ * A RaisedException represents a formula error. It includes the exception name, message, and
+ * optional details.
+ */
+export class RaisedException {
+  constructor(public name: string, public message?: string, public details?: string) {}
+
+  /**
+   * This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!"
+   */
+  public toString() {
+    switch (this.name) {
+      case 'ZeroDivisionError': return '#DIV/0!';
+      case 'UnmarshallableError': return this.details || ('#' + this.name);
+      case 'InvalidTypedValue': return `#Invalid ${this.message}: ${this.details}`;
+    }
+    return '#' + this.name;
+  }
+}
+
+/**
+ * An UnknownValue is a fallback for values that we don't handle otherwise, e.g. of a Python
+ * formula returned a function object, or a value we fail to decode.
+ * It is typically the Python repr() string of the value.
+ */
+export class UnknownValue {
+  // When encoding an unknown value, get a best-effort string form of it.
+  public static safeRepr(value: unknown): string {
+    try {
+      return String(value);
+    } catch (e) {
+      return `<${typeof value}>`;
+    }
+  }
+
+  constructor(public value: unknown) {}
+  public toString() {
+    return String(this.value);
+  }
+}
+
+/**
+ * A trivial placeholder for a value that's not yet available.
+ */
+export class PendingValue {
+  public toString() {
+    return PENDING_DATA_PLACEHOLDER;
+  }
+}
+
+/**
+ * Produces a Grist-encoded version of the value, e.g. turning a Date into ['d', timestamp].
+ * Returns ['U', repr(value)] if it fails to encode otherwise.
+ *
+ * TODO Add tests. This is not yet used for anything.
+ */
+export function encodeObject(value: unknown): CellValue {
+  try {
+    switch (typeof value) {
+      case 'string':
+      case 'number':
+      case 'boolean':
+        return value;
+    }
+    if (value == null) {
+      return null;
+    } else if (value instanceof Reference) {
+      return ['R', value.tableId, value.rowId];
+    } else if (value instanceof Date) {
+      const timestamp = value.valueOf() / 1000;
+      if ('timezone' in value) {
+        return ['D', timestamp, (value as GristDateTime).timezone];
+      } else {
+        // TODO Depending on how it's used, may want to return ['d', timestamp] for UTC midnight.
+        return ['D', timestamp, 'UTC'];
+      }
+    } else if (value instanceof RaisedException) {
+      return ['E', value.name, value.message, value.details];
+    } else if (Array.isArray(value)) {
+      return ['L', ...value.map(encodeObject)];
+    } else if (isPlainObject(value)) {
+      return ['O', mapValues(value as any, encodeObject, {sort: true})];
+    }
+  } catch (e) {
+    // Fall through to return a best-effort representation.
+  }
+  // We either don't know how to convert the value, or failed during the conversion. Instead we
+  // return an "UnmarshallableValue" object, with repr() of the value to show to the user.
+  return ['U', UnknownValue.safeRepr(value)];
+}
+
+
+/**
+ * Given a Grist-encoded value, returns an object represented by it.
+ * If the type code is unknown, or construction fails for any reason, returns an UnknownValue.
+ */
+export function decodeObject(value: CellValue): unknown {
+  if (!Array.isArray(value)) {
+    return value;
+  }
+  const code: string = value[0];
+  const args: any[] = value.slice(1);
+  let err: Error|undefined;
+  try {
+    switch (code) {
+      case 'D': return GristDateTime.fromGristValue(args[0], String(args[1]));
+      case 'd': return GristDate.fromGristValue(args[0]);
+      case 'E': return new RaisedException(args[0], args[1], args[2]);
+      case 'L': return (args as CellValue[]).map(decodeObject);
+      case 'O': return mapValues(args[0] as {[key: string]: CellValue}, decodeObject, {sort: true});
+      case 'P': return new PendingValue();
+      case 'R': return new Reference(String(args[0]), args[1]);
+      case 'U': return new UnknownValue(args[0]);
+    }
+  } catch (e) {
+    err = e;
+  }
+  // If we can't decode, return an UnknownValue with some attempt to represent what we couldn't
+  // decode as long as some info about the error if any.
+  return new UnknownValue(`${code}(${JSON.stringify(args).slice(1, -1)})` +
+    (err ? `#${err.name}(${err.message})` : ''));
+}
+
+// Like lodash's mapValues, with support for sorting keys, for friendlier output.
+export function mapValues<A, B>(
+  sourceObj: {[key: string]: A}, mapper: (value: A) => B, options: {sort?: boolean} = {}
+): {[key: string]: B} {
+  const result: {[key: string]: B} = {};
+  const keys = Object.keys(sourceObj);
+  if (options.sort) {
+    keys.sort();
+  }
+  for (const key of keys) {
+    result[key] = mapper(sourceObj[key]);
+  }
+  return result;
+}

sandbox/grist/functions/info.py

@@ -500,13 +500,12 @@ def CELL(info_type, reference):
   raise NotImplementedError()
 
 
-def RECORD(record_or_list, dates_as_iso=True, expand_refs=0):
+def RECORD(record_or_list, dates_as_iso=False, expand_refs=0):
   """
   Returns a Python dictionary with all fields in the given record. If a list of records is given,
   returns a list of corresponding Python dictionaries.
 
-  If dates_as_iso is set (which is the default), Date and DateTime values are converted to string
+  If dates_as_iso is set, Date and DateTime values are converted to string using ISO 8601 format.
-  using ISO 8601 format.
 
   If expand_refs is set to 1 or higher, Reference values are replaced with a RECORD representation
   of the referenced record, expanding the given number of levels.
@@ -540,7 +539,7 @@ def RECORD(record_or_list, dates_as_iso=True, expand_refs=0):
           for r in records]
 
 
-def _prepare_record_dict(record, dates_as_iso=True, expand_refs=0):
+def _prepare_record_dict(record, dates_as_iso=False, expand_refs=0):
   table_id = record._table.table_id
   docmodel = record._table._engine.docmodel
   columns = docmodel.get_table_rec(table_id).columns

sandbox/grist/objtypes.py

@@ -10,8 +10,8 @@ Non-primitive values are represented in actions as [type_name, args...].
 If an object cannot be encoded or decoded, an "UnmarshallableValue" is returned instead
 of the form ['U', repr(obj)].
 """
+# pylint: disable=too-many-return-statements
 import exceptions
-import marshal
 import traceback
 from datetime import date, datetime
 
@@ -50,79 +50,53 @@ class InvalidTypedValue(ValueError):
     return "Invalid %s: %s" % (self.typename, self.value)
 
 
+class AltText(object):
+  """
+  Represents a text value in a non-text column. The separate class allows formulas to access
+  wrong-type values. We use a wrapper rather than expose text directly to formulas, because with
+  text there is a risk that e.g. a formula that's supposed to add numbers would add two strings
+  with unexpected result.
+  """
+  def __init__(self, text, typename=None):
+    self._text = text
+    self._typename = typename
+
+  def __str__(self):
+    return self._text
+
+  def __int__(self):
+    # This ensures that AltText values that look like ints may be cast back to int.
+    # Convert to float first, since python does not allow casting strings with decimals to int.
+    return int(float(self._text))
+
+  def __float__(self):
+    # This ensures that AltText values that look like floats may be cast back to float.
+    return float(self._text)
+
+  def __repr__(self):
+    return '%s(%r)' % (self.__class__.__name__, self._text)
+
+  # Allow comparing to AltText("something")
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self._text == other._text
+
+  def __ne__(self, other):
+    return not self.__eq__(other)
+
+  def __hash__(self):
+    return hash((self.__class__, self._text))
+
+  def __getattr__(self, name):
+    # On attempt to do $foo.Bar on an AltText value such as "hello", raise an exception that will
+    # show up as e.g. "Invalid Ref: hello" or "Invalid Date: hello".
+    raise InvalidTypedValue(self._typename, self._text)
+
+
 _max_js_int = 1<<31
 
 def is_int_short(value):
   return -_max_js_int <= value < _max_js_int
 
-def check_marshallable(value):
-  """
-  Raises UnmarshallableError if value cannot be marshalled.
-  """
-  if isinstance(value, (str, unicode, float, bool)) or value is None:
-    # We don't need to marshal these to know they are marshallable.
-    return
-  if isinstance(value, (long, int)):
-    # Ints are also marshallable, except that we only support 32-bit ints on JS side.
-    if not is_int_short(value):
-      raise UnmarshallableError("Integer too large")
-    return
-
-  # Other things we need to try to know.
-  try:
-    marshal.dumps(value)
-  except Exception as e:
-    raise UnmarshallableError(str(e))
-
-def is_marshallable(value):
-  """
-  Returns a boolean for whether the value can be marshalled.
-  """
-  try:
-    check_marshallable(value)
-    return True
-  except Exception:
-    return False
-
-
-# Maps of type or name to (type, name, converter) tuple.
-_registered_converters_by_name = {}
-_registered_converters_by_type = {}
-
-def register_converter_by_type(type_, converter_func):
-  assert type_ not in _registered_converters_by_type
-  _registered_converters_by_type[type_] = converter_func
-
-def register_converter_by_name(converter, type_, name):
-  assert name not in _registered_converters_by_name
-  _registered_converters_by_name[name] = (type_, name, converter)
-
-
-def register_converter(converter, type_, name=None):
-  """
-  Register a new converter for the given type, with the given name (defaulting to type.__name__).
-  The converter must implement methods:
-    converter.encode_args(obj)            - should return [args...] as a python list of
-                                            marshallable arguments.
-    converter.decode_args(type, arglist)  - should return obj of type `type`.
-
-  It's up to the converter to ensure that converter.decode_args(type(obj),
-  converter.encode_args(obj)) returns a value equivalent to the original obj.
-  """
-  if name is None:
-    name = type_.__name__
-  register_converter_by_name(converter, type_, name)
-  register_converter_by_type(type_, _encode_obj_impl(converter, name))
-
-
-def deregister_converter(name):
-  """
-  De-register a named converter if previously registered.
-  """
-  prev = _registered_converters_by_name.pop(name, None)
-  if prev:
-    del _registered_converters_by_type[prev[0]]
-
 def safe_repr(obj):
   """
   Like repr(obj) but falls back to a simpler "<type-name>" string when repr() itself fails.
@@ -133,91 +107,67 @@ def safe_repr(obj):
     return '<' + type(obj).__name__ + '>'
 
 
-def encode_object(obj):
+def encode_object(value):
   """
-  Given an object, returns [typename, args...] array of marshallable values, which should be
+  Produces a Grist-encoded version of the value, e.g. turning a Date into ['d', timestamp].
-  sufficient to reconstruct `obj`. Given a primitive object, returns it unchanged.
+  Returns ['U', repr(value)] if it fails to encode otherwise.
-
-  If obj failed to encode, yields an encoding for an UnmarshallableValue object, containing
-  the repr(obj) string.
   """
   try:
-    t = type(obj)
+    if isinstance(value, (str, unicode, float, bool)) or value is None:
-    converter = (
+      return value
-        _registered_converters_by_type.get(t) or
+    elif isinstance(value, (long, int)):
-        _registered_converters_by_type[getattr(t, '_objtypes_converter_type', t)])
+      if not is_int_short(value):
-    return converter(obj)
+        raise UnmarshallableError("Integer too large")
-
+      return value
+    elif isinstance(value, AltText):
+      return str(value)
+    elif isinstance(value, records.Record):
+      return ['R', value._table.table_id, value._row_id]
+    elif isinstance(value, datetime):
+      return ['D', moment.dt_to_ts(value), value.tzinfo.zone.name if value.tzinfo else 'UTC']
+    elif isinstance(value, date):
+      return ['d', moment.date_to_ts(value)]
+    elif isinstance(value, RaisedException):
+      return ['E'] + value.encode_args()
+    elif isinstance(value, (list, tuple, RecordList)):
+      return ['L'] + [encode_object(item) for item in value]
+    elif isinstance(value, dict):
+      if not all(isinstance(key, basestring) for key in value):
+        raise UnmarshallableError("Dict with non-string keys")
+      return ['O', {key: encode_object(val) for key, val in value.iteritems()}]
   except Exception as e:
-    # We either don't know how to convert the value, or failed during the conversion. Instead we
+    pass
-    # return an "UnmarshallableValue" object, with repr() of the value to show to the user.
+  # We either don't know how to convert the value, or failed during the conversion. Instead we
-    return ['U', safe_repr(obj)]
+  # return an "UnmarshallableValue" object, with repr() of the value to show to the user.
-
+  return ['U', safe_repr(value)]
 
 def decode_object(value):
   """
-  Given a value of the form [typename, args...], returns an object represented by it. If typename
+  Given a Grist-encoded value, returns an object represented by it.
-  is unknown, or construction fails for any reason, returns (not raises!) RaisedException with
+  If typename is unknown, or construction fails for any reason, returns (not raises!)
-  original exception in its .error property.
+  RaisedException with the original exception in its .error property.
   """
-  if not isinstance(value, (tuple, list)):
-    return value
-
   try:
-    name = value[0]
+    if not isinstance(value, (list, tuple)):
+      return value
+    code = value[0]
     args = value[1:]
-    try:
+    if code == 'R':
-      type_, _, converter = _registered_converters_by_name[name]
+      return RecordStub(args[0], args[1])
-    except KeyError:
+    elif code == 'D':
-      raise KeyError("Unknown object type %r" % name)
+      return moment.ts_to_dt(args[0], moment.Zone(args[1]))
-    return converter.decode_args(type_, args)
+    elif code == 'd':
+      return moment.ts_to_date(args[0])
+    elif code == 'E':
+      return RaisedException.decode_args(*args)
+    elif code == 'L':
+      return [decode_object(item) for item in args]
+    elif code == 'O':
+      return {key: decode_object(val) for key, val in args[0].iteritems()}
+    raise KeyError("Unknown object type code %r" % code)
   except Exception as e:
     return RaisedException(e)
 
-
-class SelfConverter(object):
-  """
-  Converter for objects that implement the converter interface:
-    self.encode_args() - should return a list of marshallable arguments.
-    cls.decode_args(args...) - should return an instance given the arguments from encode_args.
-  """
-  @classmethod
-  def encode_args(cls, obj):
-    return obj.encode_args()
-
-  @classmethod
-  def decode_args(cls, type_, args):
-    return type_.decode_args(*args)
-
-#----------------------------------------------------------------------
-# Implementations of encoding objects. For basic types, there is nothing to encode, but for
-# integers, we check that they are in JS range.
-
-def _encode_obj_impl(converter, name):
-  def inner(obj):
-    args = converter.encode_args(obj)
-
-    for arg in args:
-      check_marshallable(arg)
-    return [name] + args
-  return inner
-
-def _encode_identity(value):
-  return value
-
-def _encode_integer(value):
-  if not is_int_short(value):
-    raise UnmarshallableError("Integer too large")
-  return value
-
-register_converter_by_type(str,        _encode_identity)
-register_converter_by_type(unicode,    _encode_identity)
-register_converter_by_type(float,      _encode_identity)
-register_converter_by_type(bool,       _encode_identity)
-register_converter_by_type(type(None), _encode_identity)
-register_converter_by_type(long,       _encode_integer)
-register_converter_by_type(int,        _encode_integer)
-
 #----------------------------------------------------------------------
 
 class RaisedException(object):
@@ -261,10 +211,6 @@ class RaisedException(object):
     return not self.__eq__(other)
 
 
-# Register the special wrapper class for raised exceptions with a custom short name.
-register_converter(SelfConverter, RaisedException, "E")
-
-
 class RecordList(list):
   """
   Just like list but allows setting custom attributes, which we use for remembering _group_by and
@@ -280,57 +226,6 @@ class RecordList(list):
       list.__repr__(self), self._group_by, self._sort_by)
 
 
-class ListConverter(object):
-  """
-  Converter for the 'list' type.
-  """
-  @classmethod
-  def encode_args(cls, obj):
-    return obj
-
-  @classmethod
-  def decode_args(cls, type_, args):
-    return type_(args)
-
-# Register a converter for lists, also with a custom short name. It is used, in particular, for
-# ReferenceLists. The first line ensures RecordLists are encoded as just lists; the second line
-# overrides the decoding of 'L', so that it always decodes to a plain list, since for now, at
-# least, there is no need to accept incoming RecordLists.
-register_converter_by_type(RecordList, _encode_obj_impl(ListConverter, "L"))
-register_converter(ListConverter, list, "L")
-
-
-class DateTimeConverter(object):
-  """
-  Converter for the 'datetime.datetime' type.
-  """
-  @classmethod
-  def encode_args(cls, obj):
-    return [moment.dt_to_ts(obj), obj.tzinfo.zone.name]
-
-  @classmethod
-  def decode_args(cls, _type, args):
-    return moment.ts_to_dt(args[0], moment.Zone(args[1]))
-
-# Register a converter for dates, also with a custom short name.
-register_converter(DateTimeConverter, datetime, "D")
-
-
-class DateConverter(object):
-  """
-  Converter for the 'datetime.date' type.
-  """
-  @classmethod
-  def encode_args(cls, obj):
-    return [moment.date_to_ts(obj)]
-
-  @classmethod
-  def decode_args(cls, _type, args):
-    return moment.ts_to_date(args[0])
-
-register_converter(DateConverter, date, "d")
-
-
 
 # We don't currently have a good way to convert an incoming marshalled record to a proper Record
 # object for an appropriate table. We don't expect incoming marshalled records at all, but if such
@@ -339,21 +234,3 @@ class RecordStub(object):
   def __init__(self, table_id, row_id):
     self.table_id = table_id
     self.row_id = row_id
-
-
-class RecordConverter(object):
-  """
-  Converter for 'record.Record' objects.
-  """
-  @classmethod
-  def encode_args(cls, obj):
-    return [obj._table.table_id, obj._row_id]
-
-  @classmethod
-  def decode_args(cls, _type, args):
-    return RecordStub(args[0], args[1])
-
-
-# When marshalling any subclass of Record in objtypes.py, we'll use the base Record as the type.
-records.Record._objtypes_converter_type = records.Record
-register_converter(RecordConverter, records.Record, "R")

sandbox/grist/test_record_func.py

@@ -105,21 +105,6 @@ class TestRecordFunc(test_engine.EngineTestCase):
 
     d1 = datetime.datetime(2020, 9, 13, 8, 26, 40, tzinfo=moment.tzinfo('America/New_York'))
     d2 = datetime.datetime(2017, 7, 13, 22, 40, tzinfo=moment.tzinfo('America/New_York'))
-    self.assertPartialData("Schools", ["id", "Foo"], [
-      [1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " %
-          (d1.isoformat(), d1.date().isoformat()) +
-          "'id': 1, 'name': 'Columbia'}"],
-      [2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
-          "'id': 2, 'name': 'Columbia'}"],
-      [3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " %
-          (d2.isoformat(), d2.date().isoformat()) +
-          "'id': 3, 'name': 'Yale'}"],
-      [4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
-          "'id': 4, 'name': 'Yale'}"],
-    ])
-
-    self.modify_column("Schools", "Foo",
-        formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=False))')
     self.assertPartialData("Schools", ["id", "Foo"], [
       [1, "{'address': {'city': 'New York', 'DT': %s, 'id': 11, 'D': %s}, " %
           (repr(d1), repr(d1.date())) +
@@ -133,6 +118,21 @@ class TestRecordFunc(test_engine.EngineTestCase):
           "'id': 4, 'name': 'Yale'}"],
     ])
 
+    self.modify_column("Schools", "Foo",
+        formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=True))')
+    self.assertPartialData("Schools", ["id", "Foo"], [
+      [1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " %
+          (d1.isoformat(), d1.date().isoformat()) +
+          "'id': 1, 'name': 'Columbia'}"],
+      [2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
+          "'id': 2, 'name': 'Columbia'}"],
+      [3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " %
+          (d2.isoformat(), d2.date().isoformat()) +
+          "'id': 3, 'name': 'Yale'}"],
+      [4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
+          "'id': 4, 'name': 'Yale'}"],
+    ])
+
   def test_record_set(self):
     self.load_sample(testsamples.sample_students)
     self.add_column("Students", "schools", formula='Schools.lookupRecords(name=$schoolName)')

sandbox/grist/usertypes.py

@@ -14,6 +14,7 @@ the extra complexity.
 import datetime
 import six
 import objtypes
+from objtypes import AltText
 import moment
 import logger
 from records import Record, RecordSet
@@ -59,47 +60,6 @@ def formulaType(grist_type):
     return method
   return wrapper
 
-class AltText(object):
-  """
-  Represents a text value in a non-text column. The separate class allows formulas to access
-  wrong-type values. We use a wrapper rather than expose text directly to formulas, because with
-  text there is a risk that e.g. a formula that's supposed to add numbers would add two strings
-  with unexpected result.
-  """
-  def __init__(self, text, typename=None):
-    self._text = text
-    self._typename = typename
-
-  def __str__(self):
-    return self._text
-
-  def __int__(self):
-    # This ensures that AltText values that look like ints may be cast back to int.
-    # Convert to float first, since python does not allow casting strings with decimals to int.
-    return int(float(self._text))
-
-  def __float__(self):
-    # This ensures that AltText values that look like floats may be cast back to float.
-    return float(self._text)
-
-  def __repr__(self):
-    return '%s(%r)' % (self.__class__.__name__, self._text)
-
-  # Allow comparing to AltText("something")
-  def __eq__(self, other):
-    return isinstance(other, self.__class__) and self._text == other._text
-
-  def __ne__(self, other):
-    return not self.__eq__(other)
-
-  def __hash__(self):
-    return hash((self.__class__, self._text))
-
-  def __getattr__(self, name):
-    # On attempt to do $foo.Bar on an AltText value such as "hello", raise an exception that will
-    # show up as e.g. "Invalid Ref: hello" or "Invalid Date: hello".
-    raise objtypes.InvalidTypedValue(self._typename, self._text)
-
 
 def ifError(value, value_if_error):
   """