mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) Improve object serialization, to help get RECORD data to Custom Widgets.
Summary: - Change RECORD's dates_as_str default to False. - Reimplement objtype encode_object/decode_object with less machinery. - Implement encoding of dicts (with string keys). - Make lists and dicts encode values recursively. - Implement encoding/decoding in the client - Decode automatically in plugins' fetchSelectedTable/Record, with an option to skip. Test Plan: Tested manually, not sure what tests may be affected yet. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D2593
This commit is contained in:
Dmitry S
parent
8240f8b3f0
commit
7a8debae16
@ -4,35 +4,41 @@ import {CellValue} from 'app/common/DocActions';
|
||||
import * as gristTypes from 'app/common/gristTypes';
|
||||
import * as gutil from 'app/common/gutil';
|
||||
import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
|
||||
import {decodeObject, GristDateTime} from 'app/plugin/objtypes';
|
||||
import isPlainObject = require('lodash/isPlainObject');
|
||||
import * as moment from 'moment-timezone';
|
||||
|
||||
// Some text to show on cells whose values are pending.
|
||||
export const PENDING_DATA_PLACEHOLDER = "Loading...";
|
||||
export {PENDING_DATA_PLACEHOLDER} from 'app/plugin/objtypes';
|
||||
|
||||
/**
|
||||
* Formats a custom object received as a value in a DocAction, as "Constructor(args...)".
|
||||
* E.g. ["Foo", 1, 2, 3] becomes the string "Foo(1, 2, 3)".
|
||||
* Formats a value of any type generically (with no type-specific options).
|
||||
*/
|
||||
export function formatObject(args: [string, ...any[]]): string {
|
||||
const objType = args[0], objArgs = args.slice(1);
|
||||
switch (objType) {
|
||||
case 'L': return JSON.stringify(objArgs);
|
||||
// First arg is seconds since epoch (moment takes ms), second arg is timezone
|
||||
case 'D': return moment.tz(objArgs[0] * 1000, objArgs[1]).format("YYYY-MM-DD HH:mm:ssZ");
|
||||
case 'd': return moment.tz(objArgs[0] * 1000, 'UTC').format("YYYY-MM-DD");
|
||||
case 'R': return `${objArgs[0]}[${objArgs[1]}]`;
|
||||
case 'E': return gristTypes.formatError(args);
|
||||
case 'U': return String(args[1]);
|
||||
case 'P': return PENDING_DATA_PLACEHOLDER;
|
||||
}
|
||||
return objType + "(" + JSON.stringify(objArgs).slice(1, -1) + ")";
|
||||
export function formatUnknown(value: CellValue): string {
|
||||
return formatHelper(decodeObject(value));
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a value of unknown type, using formatObject() for encoded objects.
|
||||
* Formats a decoded Grist value for displaying it. For top-level values, formats them the way we
|
||||
* like to see them in a cell or in, say, CSV export. For lists and objects, nested values are
|
||||
* formatted slighly differently, with quoted strings and ISO format for dates.
|
||||
*/
|
||||
export function formatUnknown(value: any): string {
|
||||
return gristTypes.isObject(value) ? formatObject(value) : (value == null ? "" : String(value));
|
||||
function formatHelper(value: unknown, isTopLevel: boolean = true): string {
|
||||
if (typeof value === 'object' && value) {
|
||||
if (Array.isArray(value)) {
|
||||
return '[' + value.map(v => formatHelper(v, false)).join(', ') + ']';
|
||||
} else if (isPlainObject(value)) {
|
||||
const obj: any = value;
|
||||
const items = Object.keys(obj).map(k => `${JSON.stringify(k)}: ${formatHelper(obj[k], false)}`);
|
||||
return '{' + items.join(', ') + '}';
|
||||
} else if (isTopLevel && value instanceof GristDateTime) {
|
||||
return moment(value).tz(value.timezone).format("YYYY-MM-DD HH:mm:ssZ");
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
if (isTopLevel) {
|
||||
return (value == null ? "" : String(value));
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
export type IsRightTypeFunc = (value: CellValue) => boolean;
|
||||
|
||||
@ -6,9 +6,16 @@ import isString = require('lodash/isString');
|
||||
export type GristType = 'Any' | 'Attachments' | 'Blob' | 'Bool' | 'Choice' | 'Date' | 'DateTime' |
|
||||
'Id' | 'Int' | 'ManualSortPos' | 'Numeric' | 'PositionNumber' | 'Ref' | 'RefList' | 'Text';
|
||||
|
||||
export type GristTypeInfo =
|
||||
{type: 'DateTime', timezone: string} |
|
||||
{type: 'Ref', tableId: string} |
|
||||
{type: Exclude<GristType, 'DateTime'|'Ref'>};
|
||||
|
||||
|
||||
// Letter codes for CellValue types encoded as [code, args...] tuples.
|
||||
export const enum GristObjCode {
|
||||
List = 'L',
|
||||
Dict = 'O',
|
||||
DateTime = 'D',
|
||||
Date = 'd',
|
||||
Reference = 'R',
|
||||
@ -51,6 +58,36 @@ export function getDefaultForType(colType: string, options: {sqlFormatted?: bool
|
||||
return (_defaultValues[type as GristType] || _defaultValues.Any)[options.sqlFormatted ? 1 : 0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a type like 'Numeric', 'DateTime:America/New_York', or 'Ref:Table1' to a GristTypeInfo
|
||||
* object.
|
||||
*/
|
||||
export function extractInfoFromColType(colType: string): GristTypeInfo {
|
||||
const colon = colType.indexOf(':');
|
||||
const [type, arg] = (colon === -1) ? [colType] : [colType.slice(0, colon), colType.slice(colon + 1)];
|
||||
return (type === 'Ref') ? {type, tableId: String(arg)} :
|
||||
(type === 'DateTime') ? {type, timezone: String(arg)} :
|
||||
{type} as GristTypeInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-encodes a CellValue of a given Grist type as a value suitable to use in an Any column. E.g.
|
||||
* reencodeAsAny(123, 'Numeric') -> 123
|
||||
* reencodeAsAny(123, 'Date') -> ['d', 123]
|
||||
* reencodeAsAny(123, 'Reference', 'Table1') -> ['R', 'Table1', 123]
|
||||
*/
|
||||
export function reencodeAsAny(value: CellValue, typeInfo: GristTypeInfo): CellValue {
|
||||
if (typeof value === 'number') {
|
||||
switch (typeInfo.type) {
|
||||
case 'Date': return ['d', value];
|
||||
case 'DateTime': return ['D', value, typeInfo.timezone];
|
||||
case 'Ref': return ['R', typeInfo.tableId, value];
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns whether a value (as received in a DocAction) represents a custom object.
|
||||
*/
|
||||
@ -88,20 +125,6 @@ export function isEmptyList(value: CellValue): boolean {
|
||||
return Array.isArray(value) && value.length === 1 && value[0] === GristObjCode.List;
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a raised exception (a value for which isRaisedException is true) for display in a cell.
|
||||
* This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!"
|
||||
*/
|
||||
export function formatError(value: [string, ...any[]]): string {
|
||||
const errName = value[1];
|
||||
switch (errName) {
|
||||
case 'ZeroDivisionError': return '#DIV/0!';
|
||||
case 'UnmarshallableError': return value[3] || ('#' + errName);
|
||||
case 'InvalidTypedValue': return `#Invalid ${value[2]}: ${value[3]}`;
|
||||
}
|
||||
return '#' + errName;
|
||||
}
|
||||
|
||||
function isNumber(v: CellValue) { return typeof v === 'number' || typeof v === 'boolean'; }
|
||||
function isNumberOrNull(v: CellValue) { return isNumber(v) || v === null; }
|
||||
function isBoolean(v: CellValue) { return typeof v === 'boolean' || v === 1 || v === 0; }
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
export type CellValue = number|string|boolean|null|[string, any?];
|
||||
export type CellValue = number|string|boolean|null|[string, ...unknown[]];
|
||||
|
||||
export interface RowRecord {
|
||||
id: number;
|
||||
|
||||
@ -21,6 +21,7 @@
|
||||
import { GristAPI, GristDocAPI, GristView, RPC_GRISTAPI_INTERFACE } from './GristAPI';
|
||||
import { RowRecord } from './GristData';
|
||||
import { ImportSource, ImportSourceAPI, InternalImportSourceAPI } from './InternalImportSourceAPI';
|
||||
import { decodeObject, mapValues } from './objtypes';
|
||||
import { RenderOptions, RenderTarget } from './RenderOptions';
|
||||
import { checkers } from './TypeCheckers';
|
||||
|
||||
@ -39,9 +40,26 @@ export const rpc: Rpc = new Rpc({logger: createRpcLogger()});
|
||||
export const api = rpc.getStub<GristAPI>(RPC_GRISTAPI_INTERFACE, checkers.GristAPI);
|
||||
export const coreDocApi = rpc.getStub<GristDocAPI>('GristDocAPI@grist', checkers.GristDocAPI);
|
||||
export const viewApi = rpc.getStub<GristView>('GristView', checkers.GristView);
|
||||
export const docApi = {
|
||||
|
||||
export const docApi: GristDocAPI & GristView = {
|
||||
...coreDocApi,
|
||||
...viewApi,
|
||||
|
||||
// Change fetchSelectedTable() to decode data by default, replacing e.g. ['D', timestamp] with
|
||||
// a moment date. New option `keepEncoded` skips the decoding step.
|
||||
async fetchSelectedTable(options: {keepEncoded?: boolean} = {}) {
|
||||
const table = await viewApi.fetchSelectedTable();
|
||||
return options.keepEncoded ? table :
|
||||
mapValues<any[], any[]>(table, (col) => col.map(decodeObject));
|
||||
},
|
||||
|
||||
// Change fetchSelectedRecord() to decode data by default, replacing e.g. ['D', timestamp] with
|
||||
// a moment date. New option `keepEncoded` skips the decoding step.
|
||||
async fetchSelectedRecord(rowId: number, options: {keepEncoded?: boolean} = {}) {
|
||||
const rec = await viewApi.fetchSelectedRecord(rowId);
|
||||
return options.keepEncoded ? rec :
|
||||
mapValues(rec, decodeObject);
|
||||
}
|
||||
};
|
||||
|
||||
export const on = rpc.on.bind(rpc);
|
||||
|
||||
189
app/plugin/objtypes.ts
Normal file
189
app/plugin/objtypes.ts
Normal file
@ -0,0 +1,189 @@
|
||||
/**
|
||||
* Encodes and decodes Grist encoding of values, mirroring similar Python functions in
|
||||
* sandbox/grist/objtypes.py.
|
||||
*/
|
||||
// tslint:disable:max-classes-per-file
|
||||
|
||||
import {CellValue} from 'app/plugin/GristData';
|
||||
import isPlainObject = require('lodash/isPlainObject');
|
||||
|
||||
// The text to show on cells whose values are pending.
|
||||
export const PENDING_DATA_PLACEHOLDER = "Loading...";
|
||||
|
||||
/**
|
||||
* A GristDate is just a JS Date object whose toString() method returns YYYY-MM-DD.
|
||||
*/
|
||||
export class GristDate extends Date {
|
||||
public static fromGristValue(epochSec: number): GristDate {
|
||||
return new GristDate(epochSec * 1000);
|
||||
}
|
||||
|
||||
public toString() {
|
||||
return this.toISOString().slice(0, 10);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A GristDateTime is a JS Date with an added timezone field. Its toString() returns the date in
|
||||
* ISO format. To create a timezone-aware momentjs object, use:
|
||||
*
|
||||
* moment(d).tz(d.timezone)
|
||||
*/
|
||||
export class GristDateTime extends Date {
|
||||
public static fromGristValue(epochSec: number, timezone: string): GristDateTime {
|
||||
return Object.assign(new GristDateTime(epochSec * 1000), {timezone});
|
||||
}
|
||||
|
||||
public timezone: string;
|
||||
public toString() { return this.toISOString(); }
|
||||
}
|
||||
|
||||
/**
|
||||
* A Reference represents a reference to a row in a table. It is simply a pair of a string tableId
|
||||
* and a numeric rowId.
|
||||
*/
|
||||
export class Reference {
|
||||
constructor(public tableId: string, public rowId: number) {}
|
||||
|
||||
public toString(): string {
|
||||
return `${this.tableId}[${this.rowId}]`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A RaisedException represents a formula error. It includes the exception name, message, and
|
||||
* optional details.
|
||||
*/
|
||||
export class RaisedException {
|
||||
constructor(public name: string, public message?: string, public details?: string) {}
|
||||
|
||||
/**
|
||||
* This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!"
|
||||
*/
|
||||
public toString() {
|
||||
switch (this.name) {
|
||||
case 'ZeroDivisionError': return '#DIV/0!';
|
||||
case 'UnmarshallableError': return this.details || ('#' + this.name);
|
||||
case 'InvalidTypedValue': return `#Invalid ${this.message}: ${this.details}`;
|
||||
}
|
||||
return '#' + this.name;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An UnknownValue is a fallback for values that we don't handle otherwise, e.g. of a Python
|
||||
* formula returned a function object, or a value we fail to decode.
|
||||
* It is typically the Python repr() string of the value.
|
||||
*/
|
||||
export class UnknownValue {
|
||||
// When encoding an unknown value, get a best-effort string form of it.
|
||||
public static safeRepr(value: unknown): string {
|
||||
try {
|
||||
return String(value);
|
||||
} catch (e) {
|
||||
return `<${typeof value}>`;
|
||||
}
|
||||
}
|
||||
|
||||
constructor(public value: unknown) {}
|
||||
public toString() {
|
||||
return String(this.value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A trivial placeholder for a value that's not yet available.
|
||||
*/
|
||||
export class PendingValue {
|
||||
public toString() {
|
||||
return PENDING_DATA_PLACEHOLDER;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Produces a Grist-encoded version of the value, e.g. turning a Date into ['d', timestamp].
|
||||
* Returns ['U', repr(value)] if it fails to encode otherwise.
|
||||
*
|
||||
* TODO Add tests. This is not yet used for anything.
|
||||
*/
|
||||
export function encodeObject(value: unknown): CellValue {
|
||||
try {
|
||||
switch (typeof value) {
|
||||
case 'string':
|
||||
case 'number':
|
||||
case 'boolean':
|
||||
return value;
|
||||
}
|
||||
if (value == null) {
|
||||
return null;
|
||||
} else if (value instanceof Reference) {
|
||||
return ['R', value.tableId, value.rowId];
|
||||
} else if (value instanceof Date) {
|
||||
const timestamp = value.valueOf() / 1000;
|
||||
if ('timezone' in value) {
|
||||
return ['D', timestamp, (value as GristDateTime).timezone];
|
||||
} else {
|
||||
// TODO Depending on how it's used, may want to return ['d', timestamp] for UTC midnight.
|
||||
return ['D', timestamp, 'UTC'];
|
||||
}
|
||||
} else if (value instanceof RaisedException) {
|
||||
return ['E', value.name, value.message, value.details];
|
||||
} else if (Array.isArray(value)) {
|
||||
return ['L', ...value.map(encodeObject)];
|
||||
} else if (isPlainObject(value)) {
|
||||
return ['O', mapValues(value as any, encodeObject, {sort: true})];
|
||||
}
|
||||
} catch (e) {
|
||||
// Fall through to return a best-effort representation.
|
||||
}
|
||||
// We either don't know how to convert the value, or failed during the conversion. Instead we
|
||||
// return an "UnmarshallableValue" object, with repr() of the value to show to the user.
|
||||
return ['U', UnknownValue.safeRepr(value)];
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Given a Grist-encoded value, returns an object represented by it.
|
||||
* If the type code is unknown, or construction fails for any reason, returns an UnknownValue.
|
||||
*/
|
||||
export function decodeObject(value: CellValue): unknown {
|
||||
if (!Array.isArray(value)) {
|
||||
return value;
|
||||
}
|
||||
const code: string = value[0];
|
||||
const args: any[] = value.slice(1);
|
||||
let err: Error|undefined;
|
||||
try {
|
||||
switch (code) {
|
||||
case 'D': return GristDateTime.fromGristValue(args[0], String(args[1]));
|
||||
case 'd': return GristDate.fromGristValue(args[0]);
|
||||
case 'E': return new RaisedException(args[0], args[1], args[2]);
|
||||
case 'L': return (args as CellValue[]).map(decodeObject);
|
||||
case 'O': return mapValues(args[0] as {[key: string]: CellValue}, decodeObject, {sort: true});
|
||||
case 'P': return new PendingValue();
|
||||
case 'R': return new Reference(String(args[0]), args[1]);
|
||||
case 'U': return new UnknownValue(args[0]);
|
||||
}
|
||||
} catch (e) {
|
||||
err = e;
|
||||
}
|
||||
// If we can't decode, return an UnknownValue with some attempt to represent what we couldn't
|
||||
// decode as long as some info about the error if any.
|
||||
return new UnknownValue(`${code}(${JSON.stringify(args).slice(1, -1)})` +
|
||||
(err ? `#${err.name}(${err.message})` : ''));
|
||||
}
|
||||
|
||||
// Like lodash's mapValues, with support for sorting keys, for friendlier output.
|
||||
export function mapValues<A, B>(
|
||||
sourceObj: {[key: string]: A}, mapper: (value: A) => B, options: {sort?: boolean} = {}
|
||||
): {[key: string]: B} {
|
||||
const result: {[key: string]: B} = {};
|
||||
const keys = Object.keys(sourceObj);
|
||||
if (options.sort) {
|
||||
keys.sort();
|
||||
}
|
||||
for (const key of keys) {
|
||||
result[key] = mapper(sourceObj[key]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -500,13 +500,12 @@ def CELL(info_type, reference):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def RECORD(record_or_list, dates_as_iso=True, expand_refs=0):
|
||||
def RECORD(record_or_list, dates_as_iso=False, expand_refs=0):
|
||||
"""
|
||||
Returns a Python dictionary with all fields in the given record. If a list of records is given,
|
||||
returns a list of corresponding Python dictionaries.
|
||||
|
||||
If dates_as_iso is set (which is the default), Date and DateTime values are converted to string
|
||||
using ISO 8601 format.
|
||||
If dates_as_iso is set, Date and DateTime values are converted to string using ISO 8601 format.
|
||||
|
||||
If expand_refs is set to 1 or higher, Reference values are replaced with a RECORD representation
|
||||
of the referenced record, expanding the given number of levels.
|
||||
@ -540,7 +539,7 @@ def RECORD(record_or_list, dates_as_iso=True, expand_refs=0):
|
||||
for r in records]
|
||||
|
||||
|
||||
def _prepare_record_dict(record, dates_as_iso=True, expand_refs=0):
|
||||
def _prepare_record_dict(record, dates_as_iso=False, expand_refs=0):
|
||||
table_id = record._table.table_id
|
||||
docmodel = record._table._engine.docmodel
|
||||
columns = docmodel.get_table_rec(table_id).columns
|
||||
|
||||
@ -10,8 +10,8 @@ Non-primitive values are represented in actions as [type_name, args...].
|
||||
If an object cannot be encoded or decoded, an "UnmarshallableValue" is returned instead
|
||||
of the form ['U', repr(obj)].
|
||||
"""
|
||||
# pylint: disable=too-many-return-statements
|
||||
import exceptions
|
||||
import marshal
|
||||
import traceback
|
||||
from datetime import date, datetime
|
||||
|
||||
@ -50,79 +50,53 @@ class InvalidTypedValue(ValueError):
|
||||
return "Invalid %s: %s" % (self.typename, self.value)
|
||||
|
||||
|
||||
class AltText(object):
|
||||
"""
|
||||
Represents a text value in a non-text column. The separate class allows formulas to access
|
||||
wrong-type values. We use a wrapper rather than expose text directly to formulas, because with
|
||||
text there is a risk that e.g. a formula that's supposed to add numbers would add two strings
|
||||
with unexpected result.
|
||||
"""
|
||||
def __init__(self, text, typename=None):
|
||||
self._text = text
|
||||
self._typename = typename
|
||||
|
||||
def __str__(self):
|
||||
return self._text
|
||||
|
||||
def __int__(self):
|
||||
# This ensures that AltText values that look like ints may be cast back to int.
|
||||
# Convert to float first, since python does not allow casting strings with decimals to int.
|
||||
return int(float(self._text))
|
||||
|
||||
def __float__(self):
|
||||
# This ensures that AltText values that look like floats may be cast back to float.
|
||||
return float(self._text)
|
||||
|
||||
def __repr__(self):
|
||||
return '%s(%r)' % (self.__class__.__name__, self._text)
|
||||
|
||||
# Allow comparing to AltText("something")
|
||||
def __eq__(self, other):
|
||||
return isinstance(other, self.__class__) and self._text == other._text
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.__class__, self._text))
|
||||
|
||||
def __getattr__(self, name):
|
||||
# On attempt to do $foo.Bar on an AltText value such as "hello", raise an exception that will
|
||||
# show up as e.g. "Invalid Ref: hello" or "Invalid Date: hello".
|
||||
raise InvalidTypedValue(self._typename, self._text)
|
||||
|
||||
|
||||
_max_js_int = 1<<31
|
||||
|
||||
def is_int_short(value):
|
||||
return -_max_js_int <= value < _max_js_int
|
||||
|
||||
def check_marshallable(value):
|
||||
"""
|
||||
Raises UnmarshallableError if value cannot be marshalled.
|
||||
"""
|
||||
if isinstance(value, (str, unicode, float, bool)) or value is None:
|
||||
# We don't need to marshal these to know they are marshallable.
|
||||
return
|
||||
if isinstance(value, (long, int)):
|
||||
# Ints are also marshallable, except that we only support 32-bit ints on JS side.
|
||||
if not is_int_short(value):
|
||||
raise UnmarshallableError("Integer too large")
|
||||
return
|
||||
|
||||
# Other things we need to try to know.
|
||||
try:
|
||||
marshal.dumps(value)
|
||||
except Exception as e:
|
||||
raise UnmarshallableError(str(e))
|
||||
|
||||
def is_marshallable(value):
|
||||
"""
|
||||
Returns a boolean for whether the value can be marshalled.
|
||||
"""
|
||||
try:
|
||||
check_marshallable(value)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# Maps of type or name to (type, name, converter) tuple.
|
||||
_registered_converters_by_name = {}
|
||||
_registered_converters_by_type = {}
|
||||
|
||||
def register_converter_by_type(type_, converter_func):
|
||||
assert type_ not in _registered_converters_by_type
|
||||
_registered_converters_by_type[type_] = converter_func
|
||||
|
||||
def register_converter_by_name(converter, type_, name):
|
||||
assert name not in _registered_converters_by_name
|
||||
_registered_converters_by_name[name] = (type_, name, converter)
|
||||
|
||||
|
||||
def register_converter(converter, type_, name=None):
|
||||
"""
|
||||
Register a new converter for the given type, with the given name (defaulting to type.__name__).
|
||||
The converter must implement methods:
|
||||
converter.encode_args(obj) - should return [args...] as a python list of
|
||||
marshallable arguments.
|
||||
converter.decode_args(type, arglist) - should return obj of type `type`.
|
||||
|
||||
It's up to the converter to ensure that converter.decode_args(type(obj),
|
||||
converter.encode_args(obj)) returns a value equivalent to the original obj.
|
||||
"""
|
||||
if name is None:
|
||||
name = type_.__name__
|
||||
register_converter_by_name(converter, type_, name)
|
||||
register_converter_by_type(type_, _encode_obj_impl(converter, name))
|
||||
|
||||
|
||||
def deregister_converter(name):
|
||||
"""
|
||||
De-register a named converter if previously registered.
|
||||
"""
|
||||
prev = _registered_converters_by_name.pop(name, None)
|
||||
if prev:
|
||||
del _registered_converters_by_type[prev[0]]
|
||||
|
||||
def safe_repr(obj):
|
||||
"""
|
||||
Like repr(obj) but falls back to a simpler "<type-name>" string when repr() itself fails.
|
||||
@ -133,91 +107,67 @@ def safe_repr(obj):
|
||||
return '<' + type(obj).__name__ + '>'
|
||||
|
||||
|
||||
def encode_object(obj):
|
||||
def encode_object(value):
|
||||
"""
|
||||
Given an object, returns [typename, args...] array of marshallable values, which should be
|
||||
sufficient to reconstruct `obj`. Given a primitive object, returns it unchanged.
|
||||
|
||||
If obj failed to encode, yields an encoding for an UnmarshallableValue object, containing
|
||||
the repr(obj) string.
|
||||
Produces a Grist-encoded version of the value, e.g. turning a Date into ['d', timestamp].
|
||||
Returns ['U', repr(value)] if it fails to encode otherwise.
|
||||
"""
|
||||
try:
|
||||
t = type(obj)
|
||||
converter = (
|
||||
_registered_converters_by_type.get(t) or
|
||||
_registered_converters_by_type[getattr(t, '_objtypes_converter_type', t)])
|
||||
return converter(obj)
|
||||
|
||||
if isinstance(value, (str, unicode, float, bool)) or value is None:
|
||||
return value
|
||||
elif isinstance(value, (long, int)):
|
||||
if not is_int_short(value):
|
||||
raise UnmarshallableError("Integer too large")
|
||||
return value
|
||||
elif isinstance(value, AltText):
|
||||
return str(value)
|
||||
elif isinstance(value, records.Record):
|
||||
return ['R', value._table.table_id, value._row_id]
|
||||
elif isinstance(value, datetime):
|
||||
return ['D', moment.dt_to_ts(value), value.tzinfo.zone.name if value.tzinfo else 'UTC']
|
||||
elif isinstance(value, date):
|
||||
return ['d', moment.date_to_ts(value)]
|
||||
elif isinstance(value, RaisedException):
|
||||
return ['E'] + value.encode_args()
|
||||
elif isinstance(value, (list, tuple, RecordList)):
|
||||
return ['L'] + [encode_object(item) for item in value]
|
||||
elif isinstance(value, dict):
|
||||
if not all(isinstance(key, basestring) for key in value):
|
||||
raise UnmarshallableError("Dict with non-string keys")
|
||||
return ['O', {key: encode_object(val) for key, val in value.iteritems()}]
|
||||
except Exception as e:
|
||||
# We either don't know how to convert the value, or failed during the conversion. Instead we
|
||||
# return an "UnmarshallableValue" object, with repr() of the value to show to the user.
|
||||
return ['U', safe_repr(obj)]
|
||||
|
||||
pass
|
||||
# We either don't know how to convert the value, or failed during the conversion. Instead we
|
||||
# return an "UnmarshallableValue" object, with repr() of the value to show to the user.
|
||||
return ['U', safe_repr(value)]
|
||||
|
||||
def decode_object(value):
|
||||
"""
|
||||
Given a value of the form [typename, args...], returns an object represented by it. If typename
|
||||
is unknown, or construction fails for any reason, returns (not raises!) RaisedException with
|
||||
original exception in its .error property.
|
||||
Given a Grist-encoded value, returns an object represented by it.
|
||||
If typename is unknown, or construction fails for any reason, returns (not raises!)
|
||||
RaisedException with the original exception in its .error property.
|
||||
"""
|
||||
if not isinstance(value, (tuple, list)):
|
||||
return value
|
||||
|
||||
try:
|
||||
name = value[0]
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return value
|
||||
code = value[0]
|
||||
args = value[1:]
|
||||
try:
|
||||
type_, _, converter = _registered_converters_by_name[name]
|
||||
except KeyError:
|
||||
raise KeyError("Unknown object type %r" % name)
|
||||
return converter.decode_args(type_, args)
|
||||
if code == 'R':
|
||||
return RecordStub(args[0], args[1])
|
||||
elif code == 'D':
|
||||
return moment.ts_to_dt(args[0], moment.Zone(args[1]))
|
||||
elif code == 'd':
|
||||
return moment.ts_to_date(args[0])
|
||||
elif code == 'E':
|
||||
return RaisedException.decode_args(*args)
|
||||
elif code == 'L':
|
||||
return [decode_object(item) for item in args]
|
||||
elif code == 'O':
|
||||
return {key: decode_object(val) for key, val in args[0].iteritems()}
|
||||
raise KeyError("Unknown object type code %r" % code)
|
||||
except Exception as e:
|
||||
return RaisedException(e)
|
||||
|
||||
|
||||
class SelfConverter(object):
|
||||
"""
|
||||
Converter for objects that implement the converter interface:
|
||||
self.encode_args() - should return a list of marshallable arguments.
|
||||
cls.decode_args(args...) - should return an instance given the arguments from encode_args.
|
||||
"""
|
||||
@classmethod
|
||||
def encode_args(cls, obj):
|
||||
return obj.encode_args()
|
||||
|
||||
@classmethod
|
||||
def decode_args(cls, type_, args):
|
||||
return type_.decode_args(*args)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
# Implementations of encoding objects. For basic types, there is nothing to encode, but for
|
||||
# integers, we check that they are in JS range.
|
||||
|
||||
def _encode_obj_impl(converter, name):
|
||||
def inner(obj):
|
||||
args = converter.encode_args(obj)
|
||||
|
||||
for arg in args:
|
||||
check_marshallable(arg)
|
||||
return [name] + args
|
||||
return inner
|
||||
|
||||
def _encode_identity(value):
|
||||
return value
|
||||
|
||||
def _encode_integer(value):
|
||||
if not is_int_short(value):
|
||||
raise UnmarshallableError("Integer too large")
|
||||
return value
|
||||
|
||||
register_converter_by_type(str, _encode_identity)
|
||||
register_converter_by_type(unicode, _encode_identity)
|
||||
register_converter_by_type(float, _encode_identity)
|
||||
register_converter_by_type(bool, _encode_identity)
|
||||
register_converter_by_type(type(None), _encode_identity)
|
||||
register_converter_by_type(long, _encode_integer)
|
||||
register_converter_by_type(int, _encode_integer)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
|
||||
class RaisedException(object):
|
||||
@ -261,10 +211,6 @@ class RaisedException(object):
|
||||
return not self.__eq__(other)
|
||||
|
||||
|
||||
# Register the special wrapper class for raised exceptions with a custom short name.
|
||||
register_converter(SelfConverter, RaisedException, "E")
|
||||
|
||||
|
||||
class RecordList(list):
|
||||
"""
|
||||
Just like list but allows setting custom attributes, which we use for remembering _group_by and
|
||||
@ -280,57 +226,6 @@ class RecordList(list):
|
||||
list.__repr__(self), self._group_by, self._sort_by)
|
||||
|
||||
|
||||
class ListConverter(object):
|
||||
"""
|
||||
Converter for the 'list' type.
|
||||
"""
|
||||
@classmethod
|
||||
def encode_args(cls, obj):
|
||||
return obj
|
||||
|
||||
@classmethod
|
||||
def decode_args(cls, type_, args):
|
||||
return type_(args)
|
||||
|
||||
# Register a converter for lists, also with a custom short name. It is used, in particular, for
|
||||
# ReferenceLists. The first line ensures RecordLists are encoded as just lists; the second line
|
||||
# overrides the decoding of 'L', so that it always decodes to a plain list, since for now, at
|
||||
# least, there is no need to accept incoming RecordLists.
|
||||
register_converter_by_type(RecordList, _encode_obj_impl(ListConverter, "L"))
|
||||
register_converter(ListConverter, list, "L")
|
||||
|
||||
|
||||
class DateTimeConverter(object):
|
||||
"""
|
||||
Converter for the 'datetime.datetime' type.
|
||||
"""
|
||||
@classmethod
|
||||
def encode_args(cls, obj):
|
||||
return [moment.dt_to_ts(obj), obj.tzinfo.zone.name]
|
||||
|
||||
@classmethod
|
||||
def decode_args(cls, _type, args):
|
||||
return moment.ts_to_dt(args[0], moment.Zone(args[1]))
|
||||
|
||||
# Register a converter for dates, also with a custom short name.
|
||||
register_converter(DateTimeConverter, datetime, "D")
|
||||
|
||||
|
||||
class DateConverter(object):
|
||||
"""
|
||||
Converter for the 'datetime.date' type.
|
||||
"""
|
||||
@classmethod
|
||||
def encode_args(cls, obj):
|
||||
return [moment.date_to_ts(obj)]
|
||||
|
||||
@classmethod
|
||||
def decode_args(cls, _type, args):
|
||||
return moment.ts_to_date(args[0])
|
||||
|
||||
register_converter(DateConverter, date, "d")
|
||||
|
||||
|
||||
|
||||
# We don't currently have a good way to convert an incoming marshalled record to a proper Record
|
||||
# object for an appropriate table. We don't expect incoming marshalled records at all, but if such
|
||||
@ -339,21 +234,3 @@ class RecordStub(object):
|
||||
def __init__(self, table_id, row_id):
|
||||
self.table_id = table_id
|
||||
self.row_id = row_id
|
||||
|
||||
|
||||
class RecordConverter(object):
|
||||
"""
|
||||
Converter for 'record.Record' objects.
|
||||
"""
|
||||
@classmethod
|
||||
def encode_args(cls, obj):
|
||||
return [obj._table.table_id, obj._row_id]
|
||||
|
||||
@classmethod
|
||||
def decode_args(cls, _type, args):
|
||||
return RecordStub(args[0], args[1])
|
||||
|
||||
|
||||
# When marshalling any subclass of Record in objtypes.py, we'll use the base Record as the type.
|
||||
records.Record._objtypes_converter_type = records.Record
|
||||
register_converter(RecordConverter, records.Record, "R")
|
||||
|
||||
@ -105,21 +105,6 @@ class TestRecordFunc(test_engine.EngineTestCase):
|
||||
|
||||
d1 = datetime.datetime(2020, 9, 13, 8, 26, 40, tzinfo=moment.tzinfo('America/New_York'))
|
||||
d2 = datetime.datetime(2017, 7, 13, 22, 40, tzinfo=moment.tzinfo('America/New_York'))
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " %
|
||||
(d1.isoformat(), d1.date().isoformat()) +
|
||||
"'id': 1, 'name': 'Columbia'}"],
|
||||
[2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
|
||||
"'id': 2, 'name': 'Columbia'}"],
|
||||
[3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " %
|
||||
(d2.isoformat(), d2.date().isoformat()) +
|
||||
"'id': 3, 'name': 'Yale'}"],
|
||||
[4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
|
||||
"'id': 4, 'name': 'Yale'}"],
|
||||
])
|
||||
|
||||
self.modify_column("Schools", "Foo",
|
||||
formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=False))')
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': {'city': 'New York', 'DT': %s, 'id': 11, 'D': %s}, " %
|
||||
(repr(d1), repr(d1.date())) +
|
||||
@ -133,6 +118,21 @@ class TestRecordFunc(test_engine.EngineTestCase):
|
||||
"'id': 4, 'name': 'Yale'}"],
|
||||
])
|
||||
|
||||
self.modify_column("Schools", "Foo",
|
||||
formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=True))')
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " %
|
||||
(d1.isoformat(), d1.date().isoformat()) +
|
||||
"'id': 1, 'name': 'Columbia'}"],
|
||||
[2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
|
||||
"'id': 2, 'name': 'Columbia'}"],
|
||||
[3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " %
|
||||
(d2.isoformat(), d2.date().isoformat()) +
|
||||
"'id': 3, 'name': 'Yale'}"],
|
||||
[4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
|
||||
"'id': 4, 'name': 'Yale'}"],
|
||||
])
|
||||
|
||||
def test_record_set(self):
|
||||
self.load_sample(testsamples.sample_students)
|
||||
self.add_column("Students", "schools", formula='Schools.lookupRecords(name=$schoolName)')
|
||||
|
||||
@ -14,6 +14,7 @@ the extra complexity.
|
||||
import datetime
|
||||
import six
|
||||
import objtypes
|
||||
from objtypes import AltText
|
||||
import moment
|
||||
import logger
|
||||
from records import Record, RecordSet
|
||||
@ -59,47 +60,6 @@ def formulaType(grist_type):
|
||||
return method
|
||||
return wrapper
|
||||
|
||||
class AltText(object):
|
||||
"""
|
||||
Represents a text value in a non-text column. The separate class allows formulas to access
|
||||
wrong-type values. We use a wrapper rather than expose text directly to formulas, because with
|
||||
text there is a risk that e.g. a formula that's supposed to add numbers would add two strings
|
||||
with unexpected result.
|
||||
"""
|
||||
def __init__(self, text, typename=None):
|
||||
self._text = text
|
||||
self._typename = typename
|
||||
|
||||
def __str__(self):
|
||||
return self._text
|
||||
|
||||
def __int__(self):
|
||||
# This ensures that AltText values that look like ints may be cast back to int.
|
||||
# Convert to float first, since python does not allow casting strings with decimals to int.
|
||||
return int(float(self._text))
|
||||
|
||||
def __float__(self):
|
||||
# This ensures that AltText values that look like floats may be cast back to float.
|
||||
return float(self._text)
|
||||
|
||||
def __repr__(self):
|
||||
return '%s(%r)' % (self.__class__.__name__, self._text)
|
||||
|
||||
# Allow comparing to AltText("something")
|
||||
def __eq__(self, other):
|
||||
return isinstance(other, self.__class__) and self._text == other._text
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.__class__, self._text))
|
||||
|
||||
def __getattr__(self, name):
|
||||
# On attempt to do $foo.Bar on an AltText value such as "hello", raise an exception that will
|
||||
# show up as e.g. "Invalid Ref: hello" or "Invalid Date: hello".
|
||||
raise objtypes.InvalidTypedValue(self._typename, self._text)
|
||||
|
||||
|
||||
def ifError(value, value_if_error):
|
||||
"""
|
||||
|
||||
Loading…
Reference in New Issue
Block a user