mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) Simple Python 3 compatibility changes
Summary: Changes that move towards python 3 compatibility that are easy to review without much thought Test Plan: The tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2873
This commit is contained in:
Alex Hall
parent
cc04c6481a
commit
16f297a250
@ -24,29 +24,29 @@ def get_ts_type(col_type):
|
||||
return _ts_types.get(col_type, "CellValue")
|
||||
|
||||
def main():
|
||||
print """
|
||||
print("""
|
||||
/*** THIS FILE IS AUTO-GENERATED BY %s ***/
|
||||
// tslint:disable:object-literal-key-quotes
|
||||
|
||||
export const schema = {
|
||||
""" % __file__
|
||||
""" % __file__)
|
||||
|
||||
for table in schema.schema_create_actions():
|
||||
print ' "%s": {' % table.table_id
|
||||
print(' "%s": {' % table.table_id)
|
||||
for column in table.columns:
|
||||
print ' %-20s: "%s",' % (column['id'], column['type'])
|
||||
print ' },\n'
|
||||
print(' %-20s: "%s",' % (column['id'], column['type']))
|
||||
print(' },\n')
|
||||
|
||||
print """};
|
||||
print("""};
|
||||
|
||||
export interface SchemaTypes {
|
||||
"""
|
||||
""")
|
||||
for table in schema.schema_create_actions():
|
||||
print ' "%s": {' % table.table_id
|
||||
print(' "%s": {' % table.table_id)
|
||||
for column in table.columns:
|
||||
print ' %s: %s;' % (column['id'], get_ts_type(column['type']))
|
||||
print ' };\n'
|
||||
print "}"
|
||||
print(' %s: %s;' % (column['id'], get_ts_type(column['type'])))
|
||||
print(' };\n')
|
||||
print("}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@ -67,7 +67,7 @@ def prepare_acl_table_renames(docmodel, useractions, table_renames_dict):
|
||||
if rule_info.get("tableId") in table_renames_dict:
|
||||
rule_info["tableId"] = table_renames_dict[rule_info.get("tableId")]
|
||||
rule_updates.append((rule_rec, {'userAttributes': json.dumps(rule_info)}))
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
log.warn("Error examining aclRule: %s" % (e,))
|
||||
|
||||
def do_renames():
|
||||
@ -103,7 +103,7 @@ def prepare_acl_col_renames(docmodel, useractions, col_renames_dict):
|
||||
if new_col_id:
|
||||
rule_info["lookupColId"] = new_col_id
|
||||
rule_updates.append((rule_rec, {'userAttributes': json.dumps(rule_info)}))
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
log.warn("Error examining aclRule: %s" % (e,))
|
||||
|
||||
# Go through again checking if anything in ACL formulas is affected by the rename.
|
||||
|
||||
@ -104,6 +104,11 @@ class _TreeConverter(ast.NodeVisitor):
|
||||
return ["Const", named_constants[node.id]]
|
||||
return ["Name", node.id]
|
||||
|
||||
def visit_Constant(self, node):
|
||||
return ["Const", node.value]
|
||||
|
||||
visit_NameConstant = visit_Constant
|
||||
|
||||
def visit_Attribute(self, node):
|
||||
return ["Attr", self.visit(node.value), node.attr]
|
||||
|
||||
|
||||
@ -54,9 +54,9 @@ class ActionGroup(object):
|
||||
|
||||
def get_repr(self):
|
||||
return {
|
||||
"calc": map(actions.get_action_repr, self.calc),
|
||||
"stored": map(actions.get_action_repr, self.stored),
|
||||
"undo": map(actions.get_action_repr, self.undo),
|
||||
"calc": [actions.get_action_repr(a) for a in self.calc],
|
||||
"stored": [actions.get_action_repr(a) for a in self.stored],
|
||||
"undo": [actions.get_action_repr(a) for a in self.undo],
|
||||
"direct": self.direct,
|
||||
"retValues": self.retValues
|
||||
}
|
||||
@ -64,9 +64,9 @@ class ActionGroup(object):
|
||||
@classmethod
|
||||
def from_json_obj(cls, data):
|
||||
ag = ActionGroup()
|
||||
ag.calc = map(actions.action_from_repr, data.get('calc', []))
|
||||
ag.stored = map(actions.action_from_repr, data.get('stored', []))
|
||||
ag.undo = map(actions.action_from_repr, data.get('undo', []))
|
||||
ag.calc = [actions.action_from_repr(a) for a in data.get('calc', [])]
|
||||
ag.stored = [actions.action_from_repr(a) for a in data.get('stored', [])]
|
||||
ag.undo = [actions.action_from_repr(a) for a in data.get('undo', [])]
|
||||
ag.retValues = data.get('retValues', [])
|
||||
return ag
|
||||
|
||||
|
||||
@ -4,6 +4,8 @@ It's used for collecting calculated values for formula columns.
|
||||
"""
|
||||
from collections import namedtuple
|
||||
|
||||
import six
|
||||
|
||||
import actions
|
||||
from objtypes import equal_encoding
|
||||
|
||||
@ -76,7 +78,7 @@ class ActionSummary(object):
|
||||
"""
|
||||
if not column_delta:
|
||||
return
|
||||
full_row_ids = sorted(r for r, (before, after) in column_delta.iteritems()
|
||||
full_row_ids = sorted(r for r, (before, after) in six.iteritems(column_delta)
|
||||
if not equal_encoding(before, after))
|
||||
|
||||
defunct = is_defunct(table_id) or is_defunct(col_id)
|
||||
|
||||
@ -8,6 +8,8 @@ When communicating with Node, docActions are represented as arrays [actionName,
|
||||
from collections import namedtuple
|
||||
import inspect
|
||||
|
||||
import six
|
||||
|
||||
import objtypes
|
||||
|
||||
def _eq_with_type(self, other):
|
||||
@ -86,7 +88,7 @@ def _add_simplify(SingleActionType, BulkActionType):
|
||||
else:
|
||||
def get_first(self):
|
||||
return SingleActionType(self.table_id, self.row_ids[0],
|
||||
{ key: col[0] for key, col in self.columns.iteritems()})
|
||||
{ key: col[0] for key, col in six.iteritems(self.columns)})
|
||||
def simplify(self):
|
||||
return None if not self.row_ids else (get_first(self) if len(self.row_ids) == 1 else self)
|
||||
|
||||
@ -128,7 +130,7 @@ def convert_recursive_helper(converter, data):
|
||||
return convert_recursive_helper(my_convert, data)
|
||||
"""
|
||||
if isinstance(data, dict):
|
||||
return {converter(k): converter(v) for k, v in data.iteritems()}
|
||||
return {converter(k): converter(v) for k, v in six.iteritems(data)}
|
||||
elif isinstance(data, list):
|
||||
return [converter(el) for el in data]
|
||||
elif isinstance(data, tuple):
|
||||
@ -142,10 +144,12 @@ def convert_action_values(converter, action):
|
||||
"""
|
||||
if isinstance(action, (AddRecord, UpdateRecord)):
|
||||
return type(action)(action.table_id, action.row_id,
|
||||
{k: converter(v) for k, v in action.columns.iteritems()})
|
||||
{k: converter(v) for k, v in six.iteritems(action.columns)})
|
||||
if isinstance(action, (BulkAddRecord, BulkUpdateRecord, ReplaceTableData, TableData)):
|
||||
return type(action)(action.table_id, action.row_ids,
|
||||
{k: map(converter, v) for k, v in action.columns.iteritems()})
|
||||
return type(action)(
|
||||
action.table_id, action.row_ids,
|
||||
{k: [converter(value) for value in values] for k, values in six.iteritems(action.columns)}
|
||||
)
|
||||
return action
|
||||
|
||||
def convert_recursive_in_action(converter, data):
|
||||
@ -173,7 +177,10 @@ def decode_bulk_values(bulk_values, decoder=objtypes.decode_object):
|
||||
Decode objects in values of the form {col_id: array_of_values}, as present in bulk DocActions
|
||||
and UserActions.
|
||||
"""
|
||||
return {k: map(decoder, v) for (k, v) in bulk_values.iteritems()}
|
||||
return {
|
||||
k: [decoder(value) for value in values]
|
||||
for k, values in six.iteritems(bulk_values)
|
||||
}
|
||||
|
||||
def transpose_bulk_action(bulk_action):
|
||||
"""
|
||||
|
||||
@ -4,10 +4,12 @@ Helper class for handling formula autocomplete.
|
||||
It's intended to use with rlcompleter.Completer. It allows finding global names using
|
||||
lowercase searches, and adds function usage information to some results.
|
||||
"""
|
||||
import __builtin__
|
||||
from six.moves import builtins
|
||||
import inspect
|
||||
from collections import namedtuple
|
||||
|
||||
import six
|
||||
|
||||
# funcname is the function name, e.g. "MAX"
|
||||
# argspec is the signature, e.g. "(arg, *more_args)"
|
||||
# isgrist is a boolean for whether this function should be in Grist documentation.
|
||||
@ -16,7 +18,7 @@ Completion = namedtuple('Completion', ['funcname', 'argspec', 'isgrist'])
|
||||
def is_grist_func(func):
|
||||
try:
|
||||
return inspect.getmodule(func).__name__.startswith('functions.')
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
return e
|
||||
|
||||
class AutocompleteContext(object):
|
||||
@ -24,7 +26,7 @@ class AutocompleteContext(object):
|
||||
# rlcompleter is case-sensitive. This is hard to work around while maintaining attribute
|
||||
# lookups. As a middle ground, we only introduce lowercase versions of all global names.
|
||||
self._context = {
|
||||
key: value for key, value in usercode_context.iteritems()
|
||||
key: value for key, value in six.iteritems(usercode_context)
|
||||
# Don't propose unimplemented functions in autocomplete
|
||||
if not (value and callable(value) and getattr(value, 'unimplemented', None))
|
||||
}
|
||||
@ -32,7 +34,7 @@ class AutocompleteContext(object):
|
||||
# Prepare detailed Completion objects for functions where we can supply more info.
|
||||
# TODO It would be nice to include builtin functions too, but getargspec doesn't work there.
|
||||
self._functions = {}
|
||||
for key, value in self._context.iteritems():
|
||||
for key, value in six.iteritems(self._context):
|
||||
if value and callable(value):
|
||||
argspec = inspect.formatargspec(*inspect.getargspec(value))
|
||||
self._functions[key] = Completion(key, argspec, is_grist_func(value))
|
||||
@ -47,7 +49,7 @@ class AutocompleteContext(object):
|
||||
lower = key.lower()
|
||||
if lower == key:
|
||||
continue
|
||||
if lower not in self._context and lower not in __builtin__.__dict__:
|
||||
if lower not in self._context and lower not in builtins.__dict__:
|
||||
self._lowercase[lower] = key
|
||||
else:
|
||||
# This is still good enough to find a match for, and translate back to the original.
|
||||
@ -59,7 +61,7 @@ class AutocompleteContext(object):
|
||||
self._lowercase[lower] = key
|
||||
|
||||
# Add the lowercase names to the context, and to the detailed completions in _functions.
|
||||
for lower, key in self._lowercase.iteritems():
|
||||
for lower, key in six.iteritems(self._lowercase):
|
||||
self._context[lower] = self._context[key]
|
||||
if key in self._functions:
|
||||
self._functions[lower] = self._functions[key]
|
||||
|
||||
@ -127,7 +127,7 @@ def _create_syntax_error_code(builder, input_text, err):
|
||||
def infer(node):
|
||||
try:
|
||||
return next(node.infer(), None)
|
||||
except astroid.exceptions.InferenceError, e:
|
||||
except astroid.exceptions.InferenceError as e:
|
||||
return "InferenceError on %r: %r" % (node, e)
|
||||
|
||||
|
||||
@ -137,7 +137,7 @@ def _is_table(node):
|
||||
"""
|
||||
Return true if obj is a class defining a user table.
|
||||
"""
|
||||
return (isinstance(node, astroid.nodes.Class) and node.decorators and
|
||||
return (isinstance(node, astroid.nodes.ClassDef) and node.decorators and
|
||||
node.decorators.nodes[0].as_string() == 'grist.UserTable')
|
||||
|
||||
|
||||
@ -202,7 +202,7 @@ class InferReferenceFormula(InferenceTip):
|
||||
Inference helper to treat functions decorated with `grist.formulaType(grist.Reference("Foo"))`
|
||||
as returning instances of table `Foo`.
|
||||
"""
|
||||
node_class = astroid.nodes.Function
|
||||
node_class = astroid.nodes.FunctionDef
|
||||
|
||||
@classmethod
|
||||
def filter(cls, node):
|
||||
@ -320,4 +320,4 @@ def parse_grist_names(builder):
|
||||
start, end = tok.startpos, tok.endpos
|
||||
parsed_names.append(make_tuple(start, end, obj.name, node.arg))
|
||||
|
||||
return filter(None, parsed_names)
|
||||
return [name for name in parsed_names if name]
|
||||
|
||||
@ -2,6 +2,8 @@ import json
|
||||
import types
|
||||
from collections import namedtuple
|
||||
|
||||
import six
|
||||
|
||||
import depend
|
||||
import objtypes
|
||||
import usertypes
|
||||
@ -302,7 +304,7 @@ class ChoiceListColumn(BaseColumn):
|
||||
def set(self, row_id, value):
|
||||
# When a JSON string is loaded, set it to a tuple parsed from it. When a list is loaded,
|
||||
# convert to a tuple to keep values immutable.
|
||||
if isinstance(value, basestring) and value.startswith('['):
|
||||
if isinstance(value, six.string_types) and value.startswith(u'['):
|
||||
try:
|
||||
value = tuple(json.loads(value))
|
||||
except Exception:
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import re
|
||||
import csv
|
||||
from functools import reduce
|
||||
|
||||
# Monkey-patch csv.Sniffer class, in which the quote/delimiter detection has silly bugs in the
|
||||
# regexp that it uses. It also seems poorly-implemented in other ways. We can probably do better
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
import six
|
||||
|
||||
import actions
|
||||
import schema
|
||||
import logger
|
||||
import schema
|
||||
from objtypes import strict_equal
|
||||
|
||||
log = logger.Logger(__name__, logger.INFO)
|
||||
@ -15,7 +17,7 @@ class DocActions(object):
|
||||
|
||||
def AddRecord(self, table_id, row_id, column_values):
|
||||
self.BulkAddRecord(
|
||||
table_id, [row_id], {key: [val] for key, val in column_values.iteritems()})
|
||||
table_id, [row_id], {key: [val] for key, val in six.iteritems(column_values)})
|
||||
|
||||
def BulkAddRecord(self, table_id, row_ids, column_values):
|
||||
table = self._engine.tables[table_id]
|
||||
@ -42,9 +44,9 @@ class DocActions(object):
|
||||
# Collect the undo values, and unset all values in the column (i.e. set to defaults), just to
|
||||
# make sure we don't have stale values hanging around.
|
||||
undo_values = {}
|
||||
for column in table.all_columns.itervalues():
|
||||
for column in six.itervalues(table.all_columns):
|
||||
if not column.is_private() and column.col_id != "id":
|
||||
col_values = map(column.raw_get, row_ids)
|
||||
col_values = [column.raw_get(r) for r in row_ids]
|
||||
default = column.getdefault()
|
||||
# If this column had all default values, don't include it into the undo BulkAddRecord.
|
||||
if not all(strict_equal(val, default) for val in col_values):
|
||||
@ -62,7 +64,7 @@ class DocActions(object):
|
||||
|
||||
def UpdateRecord(self, table_id, row_id, columns):
|
||||
self.BulkUpdateRecord(
|
||||
table_id, [row_id], {key: [val] for key, val in columns.iteritems()})
|
||||
table_id, [row_id], {key: [val] for key, val in six.iteritems(columns)})
|
||||
|
||||
def BulkUpdateRecord(self, table_id, row_ids, columns):
|
||||
table = self._engine.tables[table_id]
|
||||
@ -72,9 +74,9 @@ class DocActions(object):
|
||||
|
||||
# Load the updated values.
|
||||
undo_values = {}
|
||||
for col_id, values in columns.iteritems():
|
||||
for col_id, values in six.iteritems(columns):
|
||||
col = table.get_column(col_id)
|
||||
undo_values[col_id] = map(col.raw_get, row_ids)
|
||||
undo_values[col_id] = [col.raw_get(r) for r in row_ids]
|
||||
for (row_id, value) in zip(row_ids, values):
|
||||
col.set(row_id, value)
|
||||
|
||||
@ -185,7 +187,7 @@ class DocActions(object):
|
||||
log.info("ModifyColumn called which was a noop")
|
||||
return
|
||||
|
||||
undo_col_info = {k: v for k, v in schema.col_to_dict(old, include_id=False).iteritems()
|
||||
undo_col_info = {k: v for k, v in six.iteritems(schema.col_to_dict(old, include_id=False))
|
||||
if k in col_info}
|
||||
|
||||
# Remove the column from the schema, then re-add it, to force creation of a new column object.
|
||||
@ -249,7 +251,7 @@ class DocActions(object):
|
||||
|
||||
# Copy over all columns from the old table to the new.
|
||||
new_table = self._engine.tables[new_table_id]
|
||||
for new_column in new_table.all_columns.itervalues():
|
||||
for new_column in six.itervalues(new_table.all_columns):
|
||||
if not new_column.is_private():
|
||||
new_column.copy_from_column(old_table.get_column(new_column.col_id))
|
||||
new_table.grow_to_max() # We need to bring formula columns to the right size too.
|
||||
|
||||
@ -7,6 +7,8 @@ It is similar in purpose to DocModel.js on the client side.
|
||||
"""
|
||||
import itertools
|
||||
|
||||
import six
|
||||
|
||||
import records
|
||||
import usertypes
|
||||
import relabeling
|
||||
@ -107,7 +109,7 @@ def enhance_model(model_class):
|
||||
extras_class = getattr(MetaTableExtras, model_class.__name__, None)
|
||||
if not extras_class:
|
||||
return
|
||||
for name, member in extras_class.__dict__.iteritems():
|
||||
for name, member in six.iteritems(extras_class.__dict__):
|
||||
if not name.startswith("__"):
|
||||
member.__name__ = name
|
||||
member.is_private = True
|
||||
@ -238,7 +240,7 @@ class DocModel(object):
|
||||
table_obj = record_set_or_table._table
|
||||
group_by = record_set_or_table._group_by
|
||||
if group_by:
|
||||
values.update((k, [v] * count) for k, v in group_by.iteritems() if k not in values)
|
||||
values.update((k, [v] * count) for k, v in six.iteritems(group_by) if k not in values)
|
||||
else:
|
||||
table_obj = record_set_or_table.table
|
||||
|
||||
@ -281,14 +283,14 @@ def _unify_col_values(col_values, count):
|
||||
Helper that converts a dict mapping keys to values or lists of values to all lists. Non-list
|
||||
values get turned into lists by repeating them count times.
|
||||
"""
|
||||
assert all(len(v) == count for v in col_values.itervalues() if isinstance(v, list))
|
||||
assert all(len(v) == count for v in six.itervalues(col_values) if isinstance(v, list))
|
||||
return {k: (v if isinstance(v, list) else [v] * count)
|
||||
for k, v in col_values.iteritems()}
|
||||
for k, v in six.iteritems(col_values)}
|
||||
|
||||
def _get_col_values_count(col_values):
|
||||
"""
|
||||
Helper that returns the length of the first list in among the values of col_values. If none of
|
||||
the values is a list, returns 1.
|
||||
"""
|
||||
first_list = next((v for v in col_values.itervalues() if isinstance(v, list)), None)
|
||||
first_list = next((v for v in six.itervalues(col_values) if isinstance(v, list)), None)
|
||||
return len(first_list) if first_list is not None else 1
|
||||
|
||||
@ -11,6 +11,9 @@ import sys
|
||||
import time
|
||||
import traceback
|
||||
from collections import namedtuple, OrderedDict, Hashable
|
||||
|
||||
import six
|
||||
from six.moves import zip
|
||||
from sortedcontainers import SortedSet
|
||||
|
||||
import acl
|
||||
@ -34,6 +37,7 @@ import repl
|
||||
|
||||
log = logger.Logger(__name__, logger.INFO)
|
||||
|
||||
if six.PY2:
|
||||
reload(sys)
|
||||
sys.setdefaultencoding('utf8')
|
||||
|
||||
@ -260,11 +264,11 @@ class Engine(object):
|
||||
table = self.tables[data.table_id]
|
||||
|
||||
# Clear all columns, whether or not they are present in the data.
|
||||
for column in table.all_columns.itervalues():
|
||||
for column in six.itervalues(table.all_columns):
|
||||
column.clear()
|
||||
|
||||
# Only load columns that aren't stored.
|
||||
columns = {col_id: data for (col_id, data) in data.columns.iteritems()
|
||||
columns = {col_id: data for (col_id, data) in six.iteritems(data.columns)
|
||||
if table.has_column(col_id)}
|
||||
|
||||
# Add the records.
|
||||
@ -296,10 +300,10 @@ class Engine(object):
|
||||
table.grow_to_max()
|
||||
|
||||
# Load the new values.
|
||||
for col_id, values in column_values.iteritems():
|
||||
for col_id, values in six.iteritems(column_values):
|
||||
column = table.get_column(col_id)
|
||||
column.growto(growto_size)
|
||||
for row_id, value in itertools.izip(row_ids, values):
|
||||
for row_id, value in zip(row_ids, values):
|
||||
column.set(row_id, value)
|
||||
|
||||
# Invalidate new records to cause the formula columns to get recomputed.
|
||||
@ -314,16 +318,16 @@ class Engine(object):
|
||||
|
||||
query_cols = []
|
||||
if query:
|
||||
query_cols = [(table.get_column(col_id), values) for (col_id, values) in query.iteritems()]
|
||||
query_cols = [(table.get_column(col_id), values) for (col_id, values) in six.iteritems(query)]
|
||||
row_ids = [r for r in table.row_ids
|
||||
if all((c.raw_get(r) in values) for (c, values) in query_cols)]
|
||||
|
||||
for c in table.all_columns.itervalues():
|
||||
for c in six.itervalues(table.all_columns):
|
||||
# pylint: disable=too-many-boolean-expressions
|
||||
if ((formulas or not c.is_formula())
|
||||
and (private or not c.is_private())
|
||||
and c.col_id != "id" and not column.is_virtual_column(c.col_id)):
|
||||
column_values[c.col_id] = map(c.raw_get, row_ids)
|
||||
column_values[c.col_id] = [c.raw_get(r) for r in row_ids]
|
||||
|
||||
return actions.TableData(table_id, row_ids, column_values)
|
||||
|
||||
@ -355,8 +359,11 @@ class Engine(object):
|
||||
"""
|
||||
schema_actions = schema.schema_create_actions()
|
||||
table_actions = [_get_table_actions(table) for table in self.docmodel.tables.all]
|
||||
record_actions = [self._get_record_actions(table_id) for (table_id,t) in self.tables.iteritems()
|
||||
if t.next_row_id() > 1]
|
||||
record_actions = [
|
||||
self._get_record_actions(table_id)
|
||||
for (table_id,t) in six.iteritems(self.tables)
|
||||
if t.next_row_id() > 1
|
||||
]
|
||||
return schema_actions + table_actions + record_actions
|
||||
|
||||
# Returns a BulkAddRecord action which can be used to add the currently existing data to an empty
|
||||
@ -414,10 +421,10 @@ class Engine(object):
|
||||
meta_tables = self.fetch_table('_grist_Tables')
|
||||
meta_columns = self.fetch_table('_grist_Tables_column')
|
||||
gen_schema = schema.build_schema(meta_tables, meta_columns)
|
||||
gen_schema_dicts = {k: (t.tableId, dict(t.columns.iteritems()))
|
||||
for k, t in gen_schema.iteritems()}
|
||||
cur_schema_dicts = {k: (t.tableId, dict(t.columns.iteritems()))
|
||||
for k, t in self.schema.iteritems()}
|
||||
gen_schema_dicts = {k: (t.tableId, dict(t.columns))
|
||||
for k, t in six.iteritems(gen_schema)}
|
||||
cur_schema_dicts = {k: (t.tableId, dict(t.columns))
|
||||
for k, t in six.iteritems(self.schema)}
|
||||
if cur_schema_dicts != gen_schema_dicts:
|
||||
import pprint
|
||||
import difflib
|
||||
@ -448,7 +455,7 @@ class Engine(object):
|
||||
|
||||
def dump_recompute_map(self):
|
||||
log.debug("Recompute map (%d nodes):" % len(self.recompute_map))
|
||||
for node, dirty_rows in self.recompute_map.iteritems():
|
||||
for node, dirty_rows in six.iteritems(self.recompute_map):
|
||||
log.debug(" Node %s: %s" % (node, dirty_rows))
|
||||
|
||||
@contextlib.contextmanager
|
||||
@ -507,7 +514,7 @@ class Engine(object):
|
||||
Called at end of _bring_all_up_to_date or _bring_lookups_up_to_date.
|
||||
Issues actions for any accumulated cell changes.
|
||||
"""
|
||||
for node, changes in self._changes_map.iteritems():
|
||||
for node, changes in six.iteritems(self._changes_map):
|
||||
table = self.tables[node.table_id]
|
||||
col = table.get_column(node.col_id)
|
||||
# If there are changes, save them in out_actions.
|
||||
@ -876,7 +883,7 @@ class Engine(object):
|
||||
table = self.tables[action.table_id]
|
||||
new_values = {}
|
||||
extra_actions = []
|
||||
for col_id, values in column_values.iteritems():
|
||||
for col_id, values in six.iteritems(column_values):
|
||||
col_obj = table.get_column(col_id)
|
||||
values = [col_obj.convert(val) for val in values]
|
||||
|
||||
@ -895,7 +902,7 @@ class Engine(object):
|
||||
# above does it for columns explicitly mentioned; this section does it for the other
|
||||
# columns, using their default values as input to prepare_new_values().
|
||||
ignore_data = isinstance(action, actions.ReplaceTableData)
|
||||
for col_id, col_obj in table.all_columns.iteritems():
|
||||
for col_id, col_obj in six.iteritems(table.all_columns):
|
||||
if col_id in column_values or column.is_virtual_column(col_id) or col_obj.is_formula():
|
||||
continue
|
||||
defaults = [col_obj.getdefault() for r in row_ids]
|
||||
@ -922,7 +929,7 @@ class Engine(object):
|
||||
table = self.tables[action.table_id]
|
||||
|
||||
# Collect for each column the Column object and a list of new values.
|
||||
cols = [(table.get_column(col_id), values) for (col_id, values) in column_values.iteritems()]
|
||||
cols = [(table.get_column(col_id), values) for (col_id, values) in six.iteritems(column_values)]
|
||||
|
||||
# In comparisons below, we rely here on Python's "==" operator to check for equality. After a
|
||||
# type conversion, it may compare the new type to the old, e.g. 1 == 1.0 == True. It's
|
||||
@ -988,18 +995,18 @@ class Engine(object):
|
||||
old_tables = self.tables
|
||||
|
||||
self.tables = {}
|
||||
for table_id, user_table in self.gencode.usercode.__dict__.iteritems():
|
||||
for table_id, user_table in six.iteritems(self.gencode.usercode.__dict__):
|
||||
if isinstance(user_table, table_module.UserTable):
|
||||
self.tables[table_id] = (old_tables.get(table_id) or table_module.Table(table_id, self))
|
||||
|
||||
# Now update the table model for each table, and tie it to its UserTable object.
|
||||
for table_id, table in self.tables.iteritems():
|
||||
for table_id, table in six.iteritems(self.tables):
|
||||
user_table = getattr(self.gencode.usercode, table_id)
|
||||
self._update_table_model(table, user_table)
|
||||
user_table._set_table_impl(table)
|
||||
|
||||
# For any tables that are gone, use self._update_table_model to clean them up.
|
||||
for table_id, table in old_tables.iteritems():
|
||||
for table_id, table in six.iteritems(old_tables):
|
||||
if table_id not in self.tables:
|
||||
self._update_table_model(table, None)
|
||||
self._repl.locals.pop(table_id, None)
|
||||
@ -1032,8 +1039,8 @@ class Engine(object):
|
||||
table._rebuild_model(user_table)
|
||||
new_columns = table.all_columns
|
||||
|
||||
added_col_ids = new_columns.viewkeys() - old_columns.viewkeys()
|
||||
deleted_col_ids = old_columns.viewkeys() - new_columns.viewkeys()
|
||||
added_col_ids = six.viewkeys(new_columns) - six.viewkeys(old_columns)
|
||||
deleted_col_ids = six.viewkeys(old_columns) - six.viewkeys(new_columns)
|
||||
|
||||
# Invalidate the columns that got added and anything that depends on them.
|
||||
if added_col_ids:
|
||||
@ -1101,7 +1108,7 @@ class Engine(object):
|
||||
if self._schema_updated:
|
||||
self.assert_schema_consistent()
|
||||
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
# Save full exception info, so that we can rethrow accurately even if undo also fails.
|
||||
exc_info = sys.exc_info()
|
||||
# If we get an exception, we should revert all changes applied so far, to keep things
|
||||
@ -1249,7 +1256,7 @@ class Engine(object):
|
||||
(len_calc, len_stored, len_undo, len_ret) = checkpoint
|
||||
undo_actions = self.out_actions.undo[len_undo:]
|
||||
log.info("Reverting %d doc actions" % len(undo_actions))
|
||||
self.user_actions.ApplyUndoActions(map(actions.get_action_repr, undo_actions))
|
||||
self.user_actions.ApplyUndoActions([actions.get_action_repr(a) for a in undo_actions])
|
||||
del self.out_actions.calc[len_calc:]
|
||||
del self.out_actions.stored[len_stored:]
|
||||
del self.out_actions.direct[len_stored:]
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
# pylint: disable=wildcard-import
|
||||
from date import *
|
||||
from info import *
|
||||
from logical import *
|
||||
from lookup import *
|
||||
from math import *
|
||||
from stats import *
|
||||
from text import *
|
||||
from schedule import *
|
||||
from .date import *
|
||||
from .info import *
|
||||
from .logical import *
|
||||
from .lookup import *
|
||||
from .math import *
|
||||
from .stats import *
|
||||
from .text import *
|
||||
from .schedule import *
|
||||
|
||||
# Export all uppercase names, for use with `from functions import *`.
|
||||
__all__ = [k for k in dir() if not k.startswith('_') and k.isupper()]
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
import calendar
|
||||
import datetime
|
||||
import dateutil.parser
|
||||
import six
|
||||
|
||||
import moment
|
||||
import docmodel
|
||||
|
||||
@ -16,7 +18,7 @@ def _make_datetime(value):
|
||||
return datetime.datetime.combine(value, datetime.time())
|
||||
elif isinstance(value, datetime.time):
|
||||
return datetime.datetime.combine(datetime.date.today(), value)
|
||||
elif isinstance(value, basestring):
|
||||
elif isinstance(value, six.string_types):
|
||||
return dateutil.parser.parse(value)
|
||||
else:
|
||||
raise ValueError('Invalid date %r' % (value,))
|
||||
|
||||
@ -7,6 +7,8 @@ import math
|
||||
import numbers
|
||||
import re
|
||||
|
||||
import six
|
||||
|
||||
import column
|
||||
from functions import date # pylint: disable=import-error
|
||||
from functions.unimplemented import unimplemented
|
||||
@ -217,7 +219,7 @@ def ISTEXT(value):
|
||||
>>> ISTEXT(datetime.date(2011, 1, 1))
|
||||
False
|
||||
"""
|
||||
return isinstance(value, (basestring, AltText))
|
||||
return isinstance(value, (six.string_types, AltText))
|
||||
|
||||
|
||||
# Regexp for matching email. See ISEMAIL for justification.
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
from info import lazy_value_or_error, is_error
|
||||
from .info import lazy_value_or_error, is_error
|
||||
from usertypes import AltText # pylint: disable=unused-import,import-error
|
||||
|
||||
|
||||
@ -63,7 +63,7 @@ def IF(logical_expression, value_if_true, value_if_false):
|
||||
0.0
|
||||
|
||||
More tests:
|
||||
>>> IF(True, lambda: (1/0), lambda: (17))
|
||||
>>> IF(True, lambda: (1/0), lambda: (17)) # doctest: +IGNORE_EXCEPTION_DETAIL
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ZeroDivisionError: integer division or modulo by zero
|
||||
|
||||
@ -1,9 +1,10 @@
|
||||
# pylint: disable=redefined-builtin, line-too-long
|
||||
from collections import OrderedDict
|
||||
import os
|
||||
from urllib import urlencode
|
||||
import urlparse
|
||||
from unimplemented import unimplemented
|
||||
|
||||
import six
|
||||
from six.moves import urllib_parse
|
||||
from .unimplemented import unimplemented
|
||||
|
||||
@unimplemented
|
||||
def ADDRESS(row, column, absolute_relative_mode, use_a1_notation, sheet):
|
||||
@ -112,16 +113,16 @@ def SELF_HYPERLINK(label=None, page=None, **kwargs):
|
||||
if page:
|
||||
txt += "/p/{}".format(page)
|
||||
if kwargs:
|
||||
parts = list(urlparse.urlparse(txt))
|
||||
query = OrderedDict(urlparse.parse_qsl(parts[4]))
|
||||
for [key, value] in kwargs.iteritems():
|
||||
parts = list(urllib_parse.urlparse(txt))
|
||||
query = OrderedDict(urllib_parse.parse_qsl(parts[4]))
|
||||
for [key, value] in sorted(six.iteritems(kwargs)):
|
||||
key_parts = key.split('LinkKey_')
|
||||
if len(key_parts) == 2 and key_parts[0] == '':
|
||||
query[key_parts[1] + '_'] = value
|
||||
else:
|
||||
raise TypeError("unexpected keyword argument '{}' (not of form LinkKey_NAME)".format(key))
|
||||
parts[4] = urlencode(query)
|
||||
txt = urlparse.urlunparse(parts)
|
||||
parts[4] = urllib_parse.urlencode(query)
|
||||
txt = urllib_parse.urlunparse(parts)
|
||||
if label:
|
||||
txt = "{} {}".format(label, txt)
|
||||
return txt
|
||||
|
||||
@ -1,12 +1,15 @@
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
from __future__ import absolute_import
|
||||
import itertools
|
||||
import math as _math
|
||||
import operator
|
||||
import os
|
||||
import random
|
||||
import uuid
|
||||
from functools import reduce
|
||||
|
||||
from six.moves import zip, xrange
|
||||
import six
|
||||
|
||||
from functions.info import ISNUMBER, ISLOGICAL
|
||||
from functions.unimplemented import unimplemented
|
||||
@ -358,7 +361,7 @@ def INT(value):
|
||||
return int(_math.floor(value))
|
||||
|
||||
def _lcm(a, b):
|
||||
return a * b / _gcd(a, b)
|
||||
return a * b // _gcd(a, b)
|
||||
|
||||
def LCM(value1, *more_values):
|
||||
"""
|
||||
@ -790,7 +793,7 @@ def SUMPRODUCT(array1, *more_arrays):
|
||||
>>> SUMPRODUCT([-0.25, -0.25], [-2, -2], [-3, -3])
|
||||
-3.0
|
||||
"""
|
||||
return sum(reduce(operator.mul, values) for values in itertools.izip(array1, *more_arrays))
|
||||
return sum(reduce(operator.mul, values) for values in zip(array1, *more_arrays))
|
||||
|
||||
@unimplemented
|
||||
def SUMSQ(value1, value2):
|
||||
@ -842,4 +845,7 @@ def TRUNC(value, places=0):
|
||||
|
||||
def UUID():
|
||||
"""Generate a random UUID-formatted string identifier."""
|
||||
if six.PY2:
|
||||
return str(uuid.UUID(bytes=[chr(random.randrange(0, 256)) for _ in xrange(0, 16)], version=4))
|
||||
else:
|
||||
return str(uuid.UUID(bytes=bytes([random.randrange(0, 256) for _ in range(0, 16)]), version=4))
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
from datetime import datetime, timedelta
|
||||
import re
|
||||
from date import DATEADD, NOW, DTIME
|
||||
from .date import DATEADD, NOW, DTIME
|
||||
from moment_parse import MONTH_NAMES, DAY_NAMES
|
||||
|
||||
# Limit exports to schedule, so that upper-case constants like MONTH_NAMES, DAY_NAMES don't end up
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
# pylint: disable=redefined-builtin, line-too-long, unused-argument
|
||||
|
||||
from math import _chain, _chain_numeric, _chain_numeric_a
|
||||
from info import ISNUMBER, ISLOGICAL
|
||||
from date import DATE # pylint: disable=unused-import
|
||||
from unimplemented import unimplemented
|
||||
from .math import _chain, _chain_numeric, _chain_numeric_a
|
||||
from .info import ISNUMBER, ISLOGICAL
|
||||
from .date import DATE # pylint: disable=unused-import
|
||||
from .unimplemented import unimplemented
|
||||
|
||||
def _average(iterable):
|
||||
total, count = 0.0, 0
|
||||
@ -326,7 +326,7 @@ def MEDIAN(value, *more_values):
|
||||
3
|
||||
>>> MEDIAN(3, 5, 1, 4, 2)
|
||||
3
|
||||
>>> MEDIAN(xrange(10))
|
||||
>>> MEDIAN(range(10))
|
||||
4.5
|
||||
>>> MEDIAN("Hello", "123", DATE(2015, 1, 1), 12.3)
|
||||
12.3
|
||||
@ -340,9 +340,9 @@ def MEDIAN(value, *more_values):
|
||||
raise ValueError("MEDIAN requires at least one number")
|
||||
count = len(values)
|
||||
if count % 2 == 0:
|
||||
return (values[count / 2 - 1] + values[count / 2]) / 2.0
|
||||
return (values[count // 2 - 1] + values[count // 2]) / 2.0
|
||||
else:
|
||||
return values[(count - 1) / 2]
|
||||
return values[(count - 1) // 2]
|
||||
|
||||
|
||||
def MIN(value, *more_values):
|
||||
|
||||
@ -4,7 +4,7 @@ import timeit
|
||||
import unittest
|
||||
|
||||
import moment
|
||||
import schedule
|
||||
from . import schedule
|
||||
from functions.date import DTIME
|
||||
from functions import date as _date
|
||||
|
||||
@ -68,7 +68,7 @@ class TestSchedule(unittest.TestCase):
|
||||
self.assertDate(RDU(DT("2018-09-04 14:38:11"), "seconds"), "2018-09-04 14:38:11")
|
||||
self.assertDate(RDU(DT("2018-09-04 14:38:11") - TICK, "seconds"), "2018-09-04 14:38:10")
|
||||
|
||||
with self.assertRaisesRegexp(ValueError, r"Invalid unit inches"):
|
||||
with self.assertRaisesRegex(ValueError, r"Invalid unit inches"):
|
||||
RDU(DT("2018-09-04 14:38:11"), "inches")
|
||||
|
||||
def test_round_down_to_unit_tz(self):
|
||||
@ -99,11 +99,11 @@ class TestSchedule(unittest.TestCase):
|
||||
self.assertEqual(schedule._parse_interval("25-months"), (25, "months"))
|
||||
self.assertEqual(schedule._parse_interval("3-day"), (3, "days"))
|
||||
self.assertEqual(schedule._parse_interval("2-hour"), (2, "hours"))
|
||||
with self.assertRaisesRegexp(ValueError, "Not a valid interval"):
|
||||
with self.assertRaisesRegex(ValueError, "Not a valid interval"):
|
||||
schedule._parse_interval("1Year")
|
||||
with self.assertRaisesRegexp(ValueError, "Not a valid interval"):
|
||||
with self.assertRaisesRegex(ValueError, "Not a valid interval"):
|
||||
schedule._parse_interval("1y")
|
||||
with self.assertRaisesRegexp(ValueError, "Unknown unit"):
|
||||
with self.assertRaisesRegex(ValueError, "Unknown unit"):
|
||||
schedule._parse_interval("1-daily")
|
||||
|
||||
def test_parse_slot(self):
|
||||
@ -145,41 +145,41 @@ class TestSchedule(unittest.TestCase):
|
||||
|
||||
def test_parse_slot_errors(self):
|
||||
# Test failures with duplicate units
|
||||
with self.assertRaisesRegexp(ValueError, 'Duplicate unit'):
|
||||
with self.assertRaisesRegex(ValueError, 'Duplicate unit'):
|
||||
schedule._parse_slot('+1d +2d', 'weeks')
|
||||
with self.assertRaisesRegexp(ValueError, 'Duplicate unit'):
|
||||
with self.assertRaisesRegex(ValueError, 'Duplicate unit'):
|
||||
schedule._parse_slot('9:30am +2H', 'days')
|
||||
with self.assertRaisesRegexp(ValueError, 'Duplicate unit'):
|
||||
with self.assertRaisesRegex(ValueError, 'Duplicate unit'):
|
||||
schedule._parse_slot('/15 +1d', 'months')
|
||||
with self.assertRaisesRegexp(ValueError, 'Duplicate unit'):
|
||||
with self.assertRaisesRegex(ValueError, 'Duplicate unit'):
|
||||
schedule._parse_slot('Feb-1 12:30pm +20M', 'years')
|
||||
|
||||
# Test failures with improper slot types
|
||||
with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'):
|
||||
with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'):
|
||||
schedule._parse_slot('Feb-1', 'weeks')
|
||||
with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'):
|
||||
with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'):
|
||||
schedule._parse_slot('Monday', 'months')
|
||||
with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'):
|
||||
with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'):
|
||||
schedule._parse_slot('4/15', 'hours')
|
||||
with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'):
|
||||
with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'):
|
||||
schedule._parse_slot('/1', 'years')
|
||||
|
||||
# Test failures with outright invalid slot syntax.
|
||||
with self.assertRaisesRegexp(ValueError, 'Invalid slot'):
|
||||
with self.assertRaisesRegex(ValueError, 'Invalid slot'):
|
||||
schedule._parse_slot('Feb:1', 'weeks')
|
||||
with self.assertRaisesRegexp(ValueError, 'Invalid slot'):
|
||||
with self.assertRaisesRegex(ValueError, 'Invalid slot'):
|
||||
schedule._parse_slot('/1d', 'months')
|
||||
with self.assertRaisesRegexp(ValueError, 'Invalid slot'):
|
||||
with self.assertRaisesRegex(ValueError, 'Invalid slot'):
|
||||
schedule._parse_slot('10', 'hours')
|
||||
with self.assertRaisesRegexp(ValueError, 'Invalid slot'):
|
||||
with self.assertRaisesRegex(ValueError, 'Invalid slot'):
|
||||
schedule._parse_slot('H1', 'years')
|
||||
|
||||
# Test failures with unknown values
|
||||
with self.assertRaisesRegexp(ValueError, 'Unknown month'):
|
||||
with self.assertRaisesRegex(ValueError, 'Unknown month'):
|
||||
schedule._parse_slot('februarium-1', 'years')
|
||||
with self.assertRaisesRegexp(ValueError, 'Unknown day of the week'):
|
||||
with self.assertRaisesRegex(ValueError, 'Unknown day of the week'):
|
||||
schedule._parse_slot('snu', 'weeks')
|
||||
with self.assertRaisesRegexp(ValueError, 'Unknown unit'):
|
||||
with self.assertRaisesRegex(ValueError, 'Unknown unit'):
|
||||
schedule._parse_slot('+1t', 'hours')
|
||||
|
||||
def test_schedule(self):
|
||||
@ -250,14 +250,14 @@ from datetime import datetime
|
||||
]
|
||||
self.assertEqual(timing_schedule_full(), expected_result)
|
||||
t = min(timeit.repeat(stmt="t.timing_schedule_full()", setup=setup, number=N, repeat=3))
|
||||
print "\n*** SCHEDULE call with 4 points: %.2f us" % (t * 1000000 / N)
|
||||
print("\n*** SCHEDULE call with 4 points: %.2f us" % (t * 1000000 / N))
|
||||
|
||||
t = min(timeit.repeat(stmt="t.timing_schedule_init()", setup=setup, number=N, repeat=3))
|
||||
print "*** Schedule constructor: %.2f us" % (t * 1000000 / N)
|
||||
print("*** Schedule constructor: %.2f us" % (t * 1000000 / N))
|
||||
|
||||
self.assertEqual(timing_schedule_series(), expected_result)
|
||||
t = min(timeit.repeat(stmt="t.timing_schedule_series()", setup=setup, number=N, repeat=3))
|
||||
print "*** Schedule series with 4 points: %.2f us" % (t * 1000000 / N)
|
||||
print("*** Schedule series with 4 points: %.2f us" % (t * 1000000 / N))
|
||||
|
||||
def timing_schedule_full():
|
||||
return list(schedule.SCHEDULE("weekly: Mo 10:30am, We 10:30pm",
|
||||
|
||||
@ -5,7 +5,11 @@ import dateutil.parser
|
||||
import numbers
|
||||
import re
|
||||
|
||||
from unimplemented import unimplemented
|
||||
import six
|
||||
from six import unichr
|
||||
from six.moves import xrange
|
||||
|
||||
from .unimplemented import unimplemented
|
||||
from usertypes import AltText # pylint: disable=import-error
|
||||
|
||||
def CHAR(table_number):
|
||||
@ -478,7 +482,7 @@ def SUBSTITUTE(text, old_text, new_text, instance_num=None):
|
||||
if not old_text:
|
||||
return text
|
||||
|
||||
if not isinstance(new_text, basestring):
|
||||
if not isinstance(new_text, six.string_types):
|
||||
new_text = str(new_text)
|
||||
|
||||
if instance_num is None:
|
||||
|
||||
@ -19,6 +19,8 @@ import re
|
||||
import imp
|
||||
from collections import OrderedDict
|
||||
|
||||
import six
|
||||
|
||||
import codebuilder
|
||||
from column import is_visible_column
|
||||
import summary
|
||||
@ -123,7 +125,7 @@ class GenCode(object):
|
||||
source_table_id = summary.decode_summary_table_name(table_id)
|
||||
|
||||
# Sort columns by "isFormula" to output all data columns before all formula columns.
|
||||
columns = sorted(table_info.columns.itervalues(), key=lambda c: c.isFormula)
|
||||
columns = sorted(six.itervalues(table_info.columns), key=lambda c: c.isFormula)
|
||||
if filter_for_user:
|
||||
columns = [c for c in columns if is_visible_column(c.colId)]
|
||||
parts = ["@grist.UserTable\nclass %s:\n" % table_id]
|
||||
@ -136,9 +138,9 @@ class GenCode(object):
|
||||
if summary_tables:
|
||||
# Include summary formulas, for the user's information.
|
||||
formulas = OrderedDict((c.colId, c) for s in summary_tables
|
||||
for c in s.columns.itervalues() if c.isFormula)
|
||||
for c in six.itervalues(s.columns) if c.isFormula)
|
||||
parts.append(indent(textbuilder.Text("\nclass _Summary:\n")))
|
||||
for col_info in formulas.itervalues():
|
||||
for col_info in six.itervalues(formulas):
|
||||
parts.append(indent(self._make_field(col_info, table_id), levels=2))
|
||||
|
||||
return textbuilder.Combiner(parts)
|
||||
@ -147,7 +149,7 @@ class GenCode(object):
|
||||
"""Regenerates the code text and usercode module from upated document schema."""
|
||||
# Collect summary tables to group them by source table.
|
||||
summary_tables = {}
|
||||
for table_info in schema.itervalues():
|
||||
for table_info in six.itervalues(schema):
|
||||
source_table_id = summary.decode_summary_table_name(table_info.tableId)
|
||||
if source_table_id:
|
||||
summary_tables.setdefault(source_table_id, []).append(table_info)
|
||||
@ -156,7 +158,7 @@ class GenCode(object):
|
||||
"from functions import * # global uppercase functions\n" +
|
||||
"import datetime, math, re # modules commonly needed in formulas\n"]
|
||||
userparts = fullparts[:]
|
||||
for table_info in schema.itervalues():
|
||||
for table_info in six.itervalues(schema):
|
||||
fullparts.append("\n\n")
|
||||
fullparts.append(self._make_table_model(table_info, summary_tables.get(table_info.tableId)))
|
||||
if not _is_special_table(table_info.tableId):
|
||||
@ -191,5 +193,5 @@ def _is_special_table(table_id):
|
||||
def exec_module_text(module_text):
|
||||
# pylint: disable=exec-used
|
||||
mod = imp.new_module("usercode")
|
||||
exec module_text in mod.__dict__
|
||||
exec(module_text, mod.__dict__)
|
||||
return mod
|
||||
|
||||
@ -1,3 +1,6 @@
|
||||
from six.moves import xrange
|
||||
|
||||
|
||||
def _is_array(obj):
|
||||
return isinstance(obj, list)
|
||||
|
||||
|
||||
@ -287,7 +287,8 @@ class ImportActions(object):
|
||||
src_col_id = _import_transform_col_prefix + curr_col["colId"]
|
||||
log.debug("Copying from: " + src_col_id)
|
||||
|
||||
column_data[curr_col["colId"]] = map(hidden_table.get_column(src_col_id).raw_get, row_ids)
|
||||
src_col = hidden_table.get_column(src_col_id)
|
||||
column_data[curr_col["colId"]] = [src_col.raw_get(r) for r in row_ids]
|
||||
|
||||
|
||||
# ========= Cleanup, Prepare new table (if needed), insert data
|
||||
|
||||
@ -8,7 +8,7 @@ class TestMessyTables(unittest.TestCase):
|
||||
def test_any_tableset(self):
|
||||
path = os.path.join(os.path.dirname(__file__),
|
||||
"fixtures", "nyc_schools_progress_report_ec_2013.xlsx")
|
||||
with open(path, "r") as f:
|
||||
with open(path, "rb") as f:
|
||||
table_set = messytables.any.any_tableset(f, extension=os.path.splitext(path)[1])
|
||||
|
||||
self.assertIsInstance(table_set, messytables.XLSTableSet)
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
import six
|
||||
|
||||
import column
|
||||
import depend
|
||||
import records
|
||||
@ -85,7 +87,7 @@ class LookupMapColumn(column.BaseColumn):
|
||||
def _invalidate_affected(self, affected_keys):
|
||||
# For each known relation, figure out which referring rows are affected, and invalidate them.
|
||||
# The engine will notice that there have been more invalidations, and recompute things again.
|
||||
for node, rel in self._lookup_relations.iteritems():
|
||||
for node, rel in six.iteritems(self._lookup_relations):
|
||||
affected_rows = rel.get_affected_rows_by_keys(affected_keys)
|
||||
self._engine.invalidate_records(node.table_id, affected_rows, col_ids=(node.col_id,))
|
||||
|
||||
|
||||
@ -9,6 +9,8 @@ sys.path.append('thirdparty')
|
||||
import marshal
|
||||
import functools
|
||||
|
||||
import six
|
||||
|
||||
from acl_formula import parse_acl_formula
|
||||
import actions
|
||||
import sandbox
|
||||
@ -86,7 +88,7 @@ def main():
|
||||
@export
|
||||
def fetch_meta_tables(formulas=True):
|
||||
return {table_id: actions.get_action_repr(table_data)
|
||||
for (table_id, table_data) in eng.fetch_meta_tables(formulas).iteritems()}
|
||||
for (table_id, table_data) in six.iteritems(eng.fetch_meta_tables(formulas))}
|
||||
|
||||
@export
|
||||
def load_meta_tables(meta_tables, meta_columns):
|
||||
@ -100,8 +102,8 @@ def main():
|
||||
@export
|
||||
def create_migrations(all_tables, metadata_only=False):
|
||||
doc_actions = migrations.create_migrations(
|
||||
{t: table_data_from_db(t, data) for t, data in all_tables.iteritems()}, metadata_only)
|
||||
return map(actions.get_action_repr, doc_actions)
|
||||
{t: table_data_from_db(t, data) for t, data in six.iteritems(all_tables)}, metadata_only)
|
||||
return [actions.get_action_repr(action) for action in doc_actions]
|
||||
|
||||
@export
|
||||
def get_version():
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
import json
|
||||
import re
|
||||
|
||||
import six
|
||||
from six.moves import xrange
|
||||
|
||||
import actions
|
||||
import identifiers
|
||||
import schema
|
||||
@ -56,12 +59,12 @@ def create_migrations(all_tables, metadata_only=False):
|
||||
user_schema = schema.build_schema(all_tables['_grist_Tables'],
|
||||
all_tables['_grist_Tables_column'],
|
||||
include_builtin=False)
|
||||
for t in user_schema.itervalues():
|
||||
for t in six.itervalues(user_schema):
|
||||
tdset.apply_doc_action(actions.AddTable(t.tableId, schema.cols_to_dict_list(t.columns)))
|
||||
|
||||
# For each old table/column, construct an AddTable action using the current schema.
|
||||
new_schema = {a.table_id: a for a in schema.schema_create_actions()}
|
||||
for table_id, data in sorted(all_tables.iteritems()):
|
||||
for table_id, data in sorted(six.iteritems(all_tables)):
|
||||
# User tables should already be in tdset; the rest must be metadata tables.
|
||||
# (If metadata_only is true, there is simply nothing to skip here.)
|
||||
if table_id not in tdset.all_tables:
|
||||
@ -94,7 +97,7 @@ def get_last_migration_version():
|
||||
"""
|
||||
Returns the last schema version number for which we have a migration defined.
|
||||
"""
|
||||
return max(all_migrations.iterkeys())
|
||||
return max(all_migrations)
|
||||
|
||||
def migration(schema_version, need_all_tables=False):
|
||||
"""
|
||||
@ -330,16 +333,16 @@ def migration7(tdset):
|
||||
# - It doesn't fix types of Reference columns that refer to old-style summary tables
|
||||
# (if the user created some such columns manually).
|
||||
|
||||
doc_actions = filter(None, [
|
||||
doc_actions = [action for action in [
|
||||
maybe_add_column(tdset, '_grist_Tables', 'summarySourceTable', 'Ref:_grist_Tables'),
|
||||
maybe_add_column(tdset, '_grist_Tables_column', 'summarySourceCol', 'Ref:_grist_Tables_column')
|
||||
])
|
||||
] if action]
|
||||
|
||||
# Maps tableRef to Table object.
|
||||
tables_map = {t.id: t for t in actions.transpose_bulk_action(tdset.all_tables['_grist_Tables'])}
|
||||
|
||||
# Maps tableName to tableRef
|
||||
table_name_to_ref = {t.tableId: t.id for t in tables_map.itervalues()}
|
||||
table_name_to_ref = {t.tableId: t.id for t in six.itervalues(tables_map)}
|
||||
|
||||
# List of Column objects
|
||||
columns = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables_column']))
|
||||
@ -362,14 +365,14 @@ def migration7(tdset):
|
||||
# Summary tables used to be named as "Summary_<SourceName>_<ColRef1>_<ColRef2>". This regular
|
||||
# expression parses that.
|
||||
summary_re = re.compile(r'^Summary_(\w+?)((?:_\d+)*)$')
|
||||
for t in tables_map.itervalues():
|
||||
for t in six.itervalues(tables_map):
|
||||
m = summary_re.match(t.tableId)
|
||||
if not m or m.group(1) not in table_name_to_ref:
|
||||
continue
|
||||
# We have a valid summary table.
|
||||
source_table_name = m.group(1)
|
||||
source_table_ref = table_name_to_ref[source_table_name]
|
||||
groupby_colrefs = map(int, m.group(2).strip("_").split("_"))
|
||||
groupby_colrefs = [int(x) for x in m.group(2).strip("_").split("_")]
|
||||
# Prepare a new-style name for the summary table. Be sure not to conflict with existing tables
|
||||
# or with each other (i.e. don't rename multiple tables to the same name).
|
||||
new_name = summary.encode_summary_table_name(source_table_name)
|
||||
|
||||
@ -3,10 +3,11 @@ from collections import namedtuple
|
||||
import marshal
|
||||
from time import time
|
||||
import bisect
|
||||
import itertools
|
||||
import os
|
||||
import moment_parse
|
||||
import iso8601
|
||||
import six
|
||||
from six.moves import zip
|
||||
|
||||
# This is prepared by sandbox/install_tz.py
|
||||
ZoneRecord = namedtuple("ZoneRecord", ("name", "abbrs", "offsets", "untils"))
|
||||
@ -92,7 +93,7 @@ class tz(object):
|
||||
self._tzinfo = tzinfo(zonelabel)
|
||||
if isinstance(dt, datetime):
|
||||
timestamp = dt_to_ts(dt.replace(tzinfo=self._tzinfo)) * 1000
|
||||
elif isinstance(dt, (float, int, long)):
|
||||
elif isinstance(dt, (float, six.integer_types)):
|
||||
timestamp = dt
|
||||
else:
|
||||
raise TypeError("'dt' should be a datetime object or a numeric type")
|
||||
@ -181,7 +182,7 @@ class Zone(object):
|
||||
# "Until" times adjusted by the corresponding offsets. These are used in translating from
|
||||
# datetime to absolute timestamp.
|
||||
self.offset_untils = [until - offset * 60000 for (until, offset) in
|
||||
itertools.izip(self.untils, self.offsets)]
|
||||
zip(self.untils, self.offsets)]
|
||||
# Cache of TzInfo objects for this Zone, used by get_tzinfo(). There could be multiple TzInfo
|
||||
# objects, one for each possible offset, but their behavior only differs for ambiguous time.
|
||||
self._tzinfo = {}
|
||||
|
||||
@ -17,6 +17,7 @@ from math import isnan
|
||||
|
||||
import moment
|
||||
import records
|
||||
import six
|
||||
|
||||
|
||||
class UnmarshallableError(ValueError):
|
||||
@ -186,9 +187,9 @@ def encode_object(value):
|
||||
# Represent RecordSet (e.g. result of lookupRecords) in the same way as a RecordList.
|
||||
return ['L'] + [encode_object(int(item)) for item in value]
|
||||
elif isinstance(value, dict):
|
||||
if not all(isinstance(key, basestring) for key in value):
|
||||
if not all(isinstance(key, six.string_types) for key in value):
|
||||
raise UnmarshallableError("Dict with non-string keys")
|
||||
return ['O', {key: encode_object(val) for key, val in value.iteritems()}]
|
||||
return ['O', {key: encode_object(val) for key, val in six.iteritems(value)}]
|
||||
elif value == _pending_sentinel:
|
||||
return ['P']
|
||||
elif value == _censored_sentinel:
|
||||
@ -230,7 +231,7 @@ def decode_object(value):
|
||||
elif code == 'L':
|
||||
return [decode_object(item) for item in args]
|
||||
elif code == 'O':
|
||||
return {decode_object(key): decode_object(val) for key, val in args[0].iteritems()}
|
||||
return {decode_object(key): decode_object(val) for key, val in six.iteritems(args[0])}
|
||||
elif code == 'P':
|
||||
return _pending_sentinel
|
||||
elif code == 'C':
|
||||
|
||||
@ -97,6 +97,8 @@ class Record(object):
|
||||
def __nonzero__(self):
|
||||
return bool(self._row_id)
|
||||
|
||||
__bool__ = __nonzero__
|
||||
|
||||
def __repr__(self):
|
||||
return "%s[%s]" % (self._table.table_id, self._row_id)
|
||||
|
||||
@ -144,6 +146,8 @@ class RecordSet(object):
|
||||
def __nonzero__(self):
|
||||
return bool(self._row_ids)
|
||||
|
||||
__bool__ = __nonzero__
|
||||
|
||||
def __eq__(self, other):
|
||||
return (isinstance(other, RecordSet) and
|
||||
(self._table, self._row_ids, self._group_by, self._sort_by) ==
|
||||
|
||||
@ -44,6 +44,7 @@ import itertools
|
||||
import math
|
||||
import struct
|
||||
|
||||
from six.moves import zip, xrange
|
||||
from sortedcontainers import SortedList, SortedListWithKey
|
||||
|
||||
|
||||
@ -79,7 +80,7 @@ def prepare_inserts_dumb(sortedlist, keys):
|
||||
ins_groups.append((len(sortedlist), 0))
|
||||
for index, ins_count in ins_groups:
|
||||
adj_count = index - prev_index
|
||||
adjustments.extend(itertools.izip(xrange(prev_index, index),
|
||||
adjustments.extend(zip(xrange(prev_index, index),
|
||||
frange_from(next_key, adj_count)))
|
||||
next_key += adj_count
|
||||
insertions.extend(frange_from(next_key, ins_count))
|
||||
@ -233,7 +234,7 @@ class ListWithAdjustments(object):
|
||||
prev_keys.sort()
|
||||
new_keys = get_range(new_begin_key, new_end_key, count)
|
||||
|
||||
for (old_key, is_insert, i), new_key in itertools.izip(prev_keys, new_keys):
|
||||
for (old_key, is_insert, i), new_key in zip(prev_keys, new_keys):
|
||||
if is_insert:
|
||||
self._insertions.remove(old_key)
|
||||
self._insertions.add(new_key)
|
||||
@ -303,7 +304,7 @@ def all_distinct(iterable):
|
||||
"""
|
||||
a, b = itertools.tee(iterable)
|
||||
next(b, None)
|
||||
return all(x != y for x, y in itertools.izip(a, b))
|
||||
return all(x != y for x, y in zip(a, b))
|
||||
|
||||
|
||||
def range_around_float(x, i):
|
||||
|
||||
@ -57,7 +57,7 @@ class REPLInterpreter(code.InteractiveInterpreter):
|
||||
# like get/set attr and have that hurt us
|
||||
sys.stdout = user_output
|
||||
sys.stderr = user_output
|
||||
exec code in self.locals
|
||||
exec(code, self.locals)
|
||||
except:
|
||||
# bare except to catch absolutely all things the user can throw
|
||||
self.showtraceback()
|
||||
|
||||
@ -14,6 +14,8 @@ import sys
|
||||
import unittest
|
||||
sys.path.append('/thirdparty')
|
||||
|
||||
import six
|
||||
|
||||
def main():
|
||||
# Change to the directory of this file (/grist in sandbox), to discover everything under it.
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
@ -23,7 +25,9 @@ def main():
|
||||
if "--xunit" in argv:
|
||||
import xmlrunner
|
||||
argv.remove("--xunit")
|
||||
utf8_stdout = codecs.getwriter('utf8')(sys.stdout)
|
||||
utf8_stdout = sys.stdout
|
||||
if six.PY2:
|
||||
utf8_stdout = codecs.getwriter('utf8')(utf8_stdout)
|
||||
test_runner = xmlrunner.XMLTestRunner(stream=utf8_stdout)
|
||||
|
||||
if all(arg.startswith("-") for arg in argv[1:]):
|
||||
|
||||
@ -38,8 +38,8 @@ class Sandbox(object):
|
||||
|
||||
def __init__(self):
|
||||
self._functions = {}
|
||||
self._external_input = os.fdopen(3, "r", 64*1024)
|
||||
self._external_output = os.fdopen(4, "w", 64*1024)
|
||||
self._external_input = os.fdopen(3, "rb", 64*1024)
|
||||
self._external_output = os.fdopen(4, "wb", 64*1024)
|
||||
|
||||
def _send_to_js(self, msgCode, msgBody):
|
||||
# (Note that marshal version 2 is the default; we specify it explicitly for clarity. The
|
||||
|
||||
@ -10,6 +10,9 @@ Before changing this file, please review:
|
||||
|
||||
import itertools
|
||||
from collections import OrderedDict, namedtuple
|
||||
|
||||
import six
|
||||
|
||||
import actions
|
||||
|
||||
SCHEMA_VERSION = 21
|
||||
@ -310,7 +313,7 @@ def cols_to_dict_list(cols):
|
||||
|
||||
def clone_schema(schema):
|
||||
return OrderedDict((t, SchemaTable(s.tableId, s.columns.copy()))
|
||||
for (t, s) in schema.iteritems())
|
||||
for (t, s) in six.iteritems(schema))
|
||||
|
||||
def build_schema(meta_tables, meta_columns, include_builtin=True):
|
||||
"""
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
from collections import namedtuple
|
||||
import json
|
||||
import re
|
||||
|
||||
import six
|
||||
|
||||
import logger
|
||||
log = logger.Logger(__name__, logger.INFO)
|
||||
|
||||
@ -17,7 +20,7 @@ def _make_col_info(col=None, **values):
|
||||
|
||||
def _get_colinfo_dict(col_info, with_id=False):
|
||||
"""Return a dict suitable to use with AddColumn or AddTable (when with_id=True) actions."""
|
||||
col_values = {k: v for k, v in col_info._asdict().iteritems() if v is not None and k != 'colId'}
|
||||
col_values = {k: v for k, v in six.iteritems(col_info._asdict()) if v is not None and k != 'colId'}
|
||||
if with_id:
|
||||
col_values['id'] = col_info.colId
|
||||
return col_values
|
||||
@ -78,9 +81,10 @@ def _update_sort_spec(sort_spec, old_table, new_table):
|
||||
|
||||
try:
|
||||
old_sort_spec = json.loads(sort_spec)
|
||||
new_sort_spec = filter(None, [adjust(col_spec) for col_spec in old_sort_spec])
|
||||
new_sort_spec = [adjust(col_spec) for col_spec in old_sort_spec]
|
||||
new_sort_spec = [col_spec for col_spec in new_sort_spec if col_spec]
|
||||
return json.dumps(new_sort_spec, separators=(',', ':'))
|
||||
except Exception, e:
|
||||
except Exception:
|
||||
log.warn("update_summary_section: can't parse sortColRefs JSON; clearing sortColRefs")
|
||||
return ''
|
||||
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
import collections
|
||||
import types
|
||||
|
||||
import six
|
||||
from six.moves import xrange
|
||||
|
||||
import column
|
||||
import depend
|
||||
import docmodel
|
||||
@ -211,7 +214,7 @@ class Table(object):
|
||||
new_cols['id'] = self._id_column
|
||||
|
||||
# List of Columns in the same order as they appear in the generated Model definition.
|
||||
col_items = [c for c in self.Model.__dict__.iteritems() if not c[0].startswith("_")]
|
||||
col_items = [c for c in six.iteritems(self.Model.__dict__) if not c[0].startswith("_")]
|
||||
col_items.sort(key=lambda c: self._get_sort_order(c[1]))
|
||||
|
||||
for col_id, col_model in col_items:
|
||||
@ -219,11 +222,11 @@ class Table(object):
|
||||
new_cols[col_id] = self._create_or_update_col(col_id, col_model, default_func)
|
||||
|
||||
# Used for auto-completion as a record with correct properties of correct types.
|
||||
self.sample_record = _make_sample_record(self.table_id, new_cols.itervalues())
|
||||
self.sample_record = _make_sample_record(self.table_id, six.itervalues(new_cols))
|
||||
|
||||
# Note that we reuse previous special columns like lookup maps, since those not affected by
|
||||
# column changes should stay the same. These get removed when unneeded using other means.
|
||||
new_cols.update(sorted(self._special_cols.iteritems()))
|
||||
new_cols.update(sorted(six.iteritems(self._special_cols)))
|
||||
|
||||
# Set the new columns.
|
||||
self.all_columns = new_cols
|
||||
@ -289,7 +292,7 @@ class Table(object):
|
||||
"""
|
||||
return ((0, col_model._creation_order)
|
||||
if not isinstance(col_model, types.FunctionType) else
|
||||
(1, col_model.func_code.co_firstlineno))
|
||||
(1, col_model.__code__.co_firstlineno))
|
||||
|
||||
def next_row_id(self):
|
||||
"""
|
||||
@ -302,7 +305,7 @@ class Table(object):
|
||||
Resizes all columns as needed so that all valid row_ids are valid indices into all columns.
|
||||
"""
|
||||
size = self.row_ids.max() + 1
|
||||
for col_obj in self.all_columns.itervalues():
|
||||
for col_obj in six.itervalues(self.all_columns):
|
||||
col_obj.growto(size)
|
||||
|
||||
def get_column(self, col_id):
|
||||
@ -325,7 +328,7 @@ class Table(object):
|
||||
"""
|
||||
# The tuple of keys used determines the LookupMap we need.
|
||||
sort_by = kwargs.pop('sort_by', None)
|
||||
col_ids = tuple(sorted(kwargs.iterkeys()))
|
||||
col_ids = tuple(sorted(kwargs))
|
||||
key = tuple(kwargs[c] for c in col_ids)
|
||||
|
||||
lookup_map = self._get_lookup_map(col_ids)
|
||||
@ -383,7 +386,7 @@ class Table(object):
|
||||
# TODO: It should use indices, to avoid linear searching
|
||||
# TODO: It should create dependencies as needed when used from formulas.
|
||||
# TODO: It should return Record instead, for convenience of user formulas
|
||||
col_values = [(self.all_columns[col_id], value) for (col_id, value) in kwargs.iteritems()]
|
||||
col_values = [(self.all_columns[col_id], value) for (col_id, value) in six.iteritems(kwargs)]
|
||||
for row_id in self.row_ids:
|
||||
if all(col.raw_get(row_id) == value for col, value in col_values):
|
||||
return row_id
|
||||
@ -398,7 +401,7 @@ class Table(object):
|
||||
# TODO: It should use indices, to avoid linear searching
|
||||
# TODO: It should create dependencies as needed when used from formulas.
|
||||
# TODO: It should return Record instead, for convenience of user formulas
|
||||
col_values = [(self.all_columns[col_id], value) for (col_id, value) in kwargs.iteritems()]
|
||||
col_values = [(self.all_columns[col_id], value) for (col_id, value) in six.iteritems(kwargs)]
|
||||
for row_id in self.row_ids:
|
||||
if all(col.raw_get(row_id) == value for col, value in col_values):
|
||||
yield row_id
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
from itertools import izip
|
||||
from six.moves import zip as izip
|
||||
import six
|
||||
|
||||
import actions
|
||||
from usertypes import get_type_default
|
||||
|
||||
@ -29,7 +31,7 @@ class TableDataSet(object):
|
||||
def apply_doc_action(self, action):
|
||||
try:
|
||||
getattr(self, action.__class__.__name__)(*action)
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
log.warn("ERROR applying action %s: %s" % (action, e))
|
||||
raise
|
||||
|
||||
@ -48,12 +50,12 @@ class TableDataSet(object):
|
||||
# Actions on records.
|
||||
#----------------------------------------
|
||||
def AddRecord(self, table_id, row_id, columns):
|
||||
self.BulkAddRecord(table_id, [row_id], {key: [val] for key, val in columns.iteritems()})
|
||||
self.BulkAddRecord(table_id, [row_id], {key: [val] for key, val in six.iteritems(columns)})
|
||||
|
||||
def BulkAddRecord(self, table_id, row_ids, columns):
|
||||
table_data = self.all_tables[table_id]
|
||||
table_data.row_ids.extend(row_ids)
|
||||
for col, values in table_data.columns.iteritems():
|
||||
for col, values in six.iteritems(table_data.columns):
|
||||
if col in columns:
|
||||
values.extend(columns[col])
|
||||
else:
|
||||
@ -67,19 +69,19 @@ class TableDataSet(object):
|
||||
def BulkRemoveRecord(self, table_id, row_ids):
|
||||
table_data = self.all_tables[table_id]
|
||||
remove_set = set(row_ids)
|
||||
for col, values in table_data.columns.iteritems():
|
||||
for col, values in six.iteritems(table_data.columns):
|
||||
values[:] = [v for r, v in izip(table_data.row_ids, values) if r not in remove_set]
|
||||
table_data.row_ids[:] = [r for r in table_data.row_ids if r not in remove_set]
|
||||
|
||||
def UpdateRecord(self, table_id, row_id, columns):
|
||||
self.BulkUpdateRecord(
|
||||
table_id, [row_id], {key: [val] for key, val in columns.iteritems()})
|
||||
table_id, [row_id], {key: [val] for key, val in six.iteritems(columns)})
|
||||
|
||||
def BulkUpdateRecord(self, table_id, row_ids, columns):
|
||||
table_data = self.all_tables[table_id]
|
||||
rowid_map = {r:i for i, r in enumerate(table_data.row_ids)}
|
||||
table_indices = [rowid_map[r] for r in row_ids]
|
||||
for col, values in columns.iteritems():
|
||||
for col, values in six.iteritems(columns):
|
||||
if col in table_data.columns:
|
||||
col_values = table_data.columns[col]
|
||||
for i, v in izip(table_indices, values):
|
||||
@ -88,7 +90,7 @@ class TableDataSet(object):
|
||||
def ReplaceTableData(self, table_id, row_ids, columns):
|
||||
table_data = self.all_tables[table_id]
|
||||
del table_data.row_ids[:]
|
||||
for col, values in table_data.columns.iteritems():
|
||||
for col, values in six.iteritems(table_data.columns):
|
||||
del values[:]
|
||||
self.BulkAddRecord(table_id, row_ids, columns)
|
||||
|
||||
|
||||
@ -5,6 +5,7 @@ import unittest
|
||||
from acl_formula import parse_acl_formula
|
||||
import test_engine
|
||||
|
||||
|
||||
class TestACLFormula(unittest.TestCase):
|
||||
def test_basic(self):
|
||||
# Test a few basic formulas and structures, hitting everything we expect to support
|
||||
@ -104,19 +105,19 @@ class TestACLFormula(unittest.TestCase):
|
||||
self.assertRaises(SyntaxError, parse_acl_formula, "def foo(): pass")
|
||||
|
||||
# Unsupported node type
|
||||
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "max(rec)")
|
||||
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "user.id in {1, 2, 3}")
|
||||
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "1 if user.IsAnon else 2")
|
||||
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "max(rec)")
|
||||
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "user.id in {1, 2, 3}")
|
||||
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 if user.IsAnon else 2")
|
||||
|
||||
# Unsupported operation
|
||||
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "1 | 2")
|
||||
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "1 << 2")
|
||||
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "~test")
|
||||
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 | 2")
|
||||
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 << 2")
|
||||
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "~test")
|
||||
|
||||
# Syntax error
|
||||
self.assertRaises(SyntaxError, parse_acl_formula, "[(]")
|
||||
self.assertRaises(SyntaxError, parse_acl_formula, "user.id in (1,2))")
|
||||
self.assertRaisesRegexp(SyntaxError, r'invalid syntax on line 1 col 9', parse_acl_formula, "foo and !bar")
|
||||
self.assertRaisesRegex(SyntaxError, r'invalid syntax on line 1 col 9', parse_acl_formula, "foo and !bar")
|
||||
|
||||
class TestACLFormulaUserActions(test_engine.EngineTestCase):
|
||||
def test_acl_actions(self):
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
import unittest
|
||||
|
||||
import codebuilder
|
||||
import six
|
||||
|
||||
|
||||
def make_body(formula, default=None):
|
||||
return codebuilder.make_formula_body(formula, default).get_text()
|
||||
@ -122,7 +124,7 @@ for a in rec:
|
||||
return rec
|
||||
""")
|
||||
|
||||
self.assertRegexpMatches(make_body(body),
|
||||
self.assertRegex(make_body(body),
|
||||
r"raise SyntaxError\('Grist disallows assignment" +
|
||||
r" to the special variable \"rec\" on line 4 col 7'\)")
|
||||
|
||||
@ -135,8 +137,8 @@ return rec
|
||||
self.assertEqual(make_body(u"'résumé' + $foo"), u"return 'résumé' + rec.foo")
|
||||
|
||||
# Check the return type of make_body()
|
||||
self.assertEqual(type(make_body("foo")), unicode)
|
||||
self.assertEqual(type(make_body(u"foo")), unicode)
|
||||
self.assertEqual(type(make_body("foo")), six.text_type)
|
||||
self.assertEqual(type(make_body(u"foo")), six.text_type)
|
||||
|
||||
|
||||
def test_wrap_logical(self):
|
||||
|
||||
@ -338,7 +338,7 @@ class TestColumnActions(test_engine.EngineTestCase):
|
||||
self.init_sample_data()
|
||||
|
||||
# Test that we cannot remove group-by columns from summary tables directly.
|
||||
with self.assertRaisesRegexp(ValueError, "cannot remove .* group-by"):
|
||||
with self.assertRaisesRegex(ValueError, "cannot remove .* group-by"):
|
||||
self.apply_user_action(["BulkRemoveRecord", '_grist_Tables_column', [20,18]])
|
||||
|
||||
# Test that group-by columns in summary tables get removed.
|
||||
|
||||
@ -244,6 +244,6 @@ class TestDocModel(test_engine.EngineTestCase):
|
||||
|
||||
# Verify that positions are set such that the order is what we asked for.
|
||||
student_columns = self.engine.docmodel.tables.lookupOne(tableId='Students').columns
|
||||
self.assertEqual(map(int, student_columns), [1,2,4,5,6,25,22,23])
|
||||
self.assertEqual(list(map(int, student_columns)), [1,2,4,5,6,25,22,23])
|
||||
school_columns = self.engine.docmodel.tables.lookupOne(tableId='Schools').columns
|
||||
self.assertEqual(map(int, school_columns), [24,10,12])
|
||||
self.assertEqual(list(map(int, school_columns)), [24,10,12])
|
||||
|
||||
@ -4,6 +4,8 @@ import json
|
||||
import unittest
|
||||
from collections import namedtuple
|
||||
|
||||
import six
|
||||
|
||||
import actions
|
||||
import column
|
||||
import engine
|
||||
@ -22,6 +24,8 @@ View = namedtuple('View', 'id sections')
|
||||
Section = namedtuple('Section', 'id parentKey tableRef fields')
|
||||
Field = namedtuple('Field', 'id colRef')
|
||||
|
||||
unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
|
||||
unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
|
||||
|
||||
class EngineTestCase(unittest.TestCase):
|
||||
"""
|
||||
@ -152,9 +156,9 @@ class EngineTestCase(unittest.TestCase):
|
||||
# Convert observed and expected actions into a comparable form.
|
||||
for k in self.action_group_action_fields:
|
||||
if k in observed:
|
||||
observed[k] = map(get_comparable_repr, observed[k])
|
||||
observed[k] = [get_comparable_repr(v) for v in observed[k]]
|
||||
if k in expected:
|
||||
expected[k] = map(get_comparable_repr, expected[k])
|
||||
expected[k] = [get_comparable_repr(v) for v in expected[k]]
|
||||
|
||||
if observed != expected:
|
||||
o_lines = self._formatActionGroup(observed)
|
||||
@ -192,13 +196,13 @@ class EngineTestCase(unittest.TestCase):
|
||||
output = {t: self.engine.fetch_table(t) for t in self.engine.schema}
|
||||
output = testutil.replace_nans(output)
|
||||
output = actions.encode_objects(output)
|
||||
print ''.join(self._getEngineDataLines(output))
|
||||
print(''.join(self._getEngineDataLines(output)))
|
||||
|
||||
def dump_actions(self, out_actions):
|
||||
"""
|
||||
Prints out_actions in human-readable format, for help in writing / debugging tets.
|
||||
"""
|
||||
print "\n".join(self._formatActionGroup(out_actions.__dict__))
|
||||
print("\n".join(self._formatActionGroup(out_actions.__dict__)))
|
||||
|
||||
def assertTableData(self, table_name, data=[], cols="all", rows="all", sort=None):
|
||||
"""
|
||||
@ -237,7 +241,7 @@ class EngineTestCase(unittest.TestCase):
|
||||
if sort:
|
||||
row_ids.sort(key=lambda r: sort(table.get_record(r)))
|
||||
|
||||
observed_col_data = {c.col_id: map(c.raw_get, row_ids) for c in columns if c.col_id != "id"}
|
||||
observed_col_data = {c.col_id: [c.raw_get(r) for r in row_ids] for c in columns if c.col_id != "id"}
|
||||
observed = actions.TableData(table_name, row_ids, observed_col_data)
|
||||
self.assertEqualDocData({table_name: observed}, {table_name: expected},
|
||||
col_names=col_names)
|
||||
@ -286,7 +290,7 @@ class EngineTestCase(unittest.TestCase):
|
||||
"""
|
||||
schema = sample["SCHEMA"]
|
||||
self.engine.load_meta_tables(schema['_grist_Tables'], schema['_grist_Tables_column'])
|
||||
for data in sample["DATA"].itervalues():
|
||||
for data in six.itervalues(sample["DATA"]):
|
||||
self.engine.load_table(data)
|
||||
self.engine.load_done()
|
||||
|
||||
@ -426,11 +430,11 @@ class TestEngine(EngineTestCase):
|
||||
sample = self.samples[data.pop("USE_SAMPLE")]
|
||||
expected_data = sample["DATA"].copy()
|
||||
expected_data.update({t: testutil.table_data_from_rows(t, tdata[0], tdata[1:])
|
||||
for (t, tdata) in data.iteritems()})
|
||||
for (t, tdata) in six.iteritems(data)})
|
||||
self.assertCorrectEngineData(expected_data)
|
||||
else:
|
||||
raise ValueError("Unrecognized step %s in test script" % step)
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
prefix = "LINE %s: " % line
|
||||
e.args = (prefix + e.args[0],) + e.args[1:] if e.args else (prefix,)
|
||||
raise
|
||||
@ -526,7 +530,7 @@ class TestEngine(EngineTestCase):
|
||||
# Simulate an error inside a DocAction, and make sure we restore the schema (don't leave it in
|
||||
# inconsistent with metadata).
|
||||
self.load_sample(testutil.parse_test_sample(self.sample1))
|
||||
with self.assertRaisesRegexp(AttributeError, r"'BAD'"):
|
||||
with self.assertRaisesRegex(AttributeError, r"'BAD'"):
|
||||
self.add_column('Address', 'bad', isFormula=False, type="BAD")
|
||||
self.engine.assert_schema_consistent()
|
||||
|
||||
|
||||
@ -2,6 +2,9 @@
|
||||
Tests that formula error messages (traceback) are correct
|
||||
"""
|
||||
import textwrap
|
||||
|
||||
import six
|
||||
|
||||
import depend
|
||||
import test_engine
|
||||
import testutil
|
||||
@ -48,14 +51,19 @@ else:
|
||||
self.assertIsInstance(exc.error, type_)
|
||||
self.assertEqual(str(exc.error), message)
|
||||
if tracebackRegexp:
|
||||
self.assertRegexpMatches(exc.details, tracebackRegexp)
|
||||
self.assertRegex(exc.details, tracebackRegexp)
|
||||
|
||||
def test_formula_errors(self):
|
||||
self.load_sample(self.sample)
|
||||
|
||||
if six.PY2:
|
||||
self.assertFormulaError(self.engine.get_formula_error('Math', 'excel_formula', 3),
|
||||
TypeError, 'SQRT() takes exactly 1 argument (2 given)',
|
||||
r"TypeError: SQRT\(\) takes exactly 1 argument \(2 given\)")
|
||||
else:
|
||||
self.assertFormulaError(self.engine.get_formula_error('Math', 'excel_formula', 3),
|
||||
TypeError, 'SQRT() takes 1 positional argument but 2 were given',
|
||||
r"TypeError: SQRT\(\) takes 1 positional argument but 2 were given")
|
||||
|
||||
self.assertFormulaError(self.engine.get_formula_error('Math', 'built_in_formula', 3),
|
||||
TypeError, "'int' object is not iterable")
|
||||
|
||||
@ -2,6 +2,8 @@ import unittest
|
||||
import difflib
|
||||
import re
|
||||
|
||||
from six.moves import xrange
|
||||
|
||||
import gencode
|
||||
import identifiers
|
||||
import records
|
||||
|
||||
@ -36,10 +36,10 @@ class TestGpath(unittest.TestCase):
|
||||
self.assertEqual(self.obj["hello"], "blah")
|
||||
|
||||
def test_set_strict(self):
|
||||
with self.assertRaisesRegexp(Exception, r"non-existent"):
|
||||
with self.assertRaisesRegex(Exception, r"non-existent"):
|
||||
gpath.place(self.obj, ["bar", 4], 17)
|
||||
|
||||
with self.assertRaisesRegexp(Exception, r"not a plain object"):
|
||||
with self.assertRaisesRegex(Exception, r"not a plain object"):
|
||||
gpath.place(self.obj, ["foo", 0], 17)
|
||||
|
||||
|
||||
@ -54,13 +54,13 @@ class TestGpath(unittest.TestCase):
|
||||
["asdf", {"bar": 1}, {"bar": 2}, "hello", {"baz": 3}, "world"])
|
||||
|
||||
def test_insert_strict(self):
|
||||
with self.assertRaisesRegexp(Exception, r'not an array'):
|
||||
with self.assertRaisesRegex(Exception, r'not an array'):
|
||||
gpath.insert(self.obj, ["foo"], "asdf")
|
||||
|
||||
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
|
||||
with self.assertRaisesRegex(Exception, r'invalid.*index'):
|
||||
gpath.insert(self.obj, ["foo", -1], 17)
|
||||
|
||||
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
|
||||
with self.assertRaisesRegex(Exception, r'invalid.*index'):
|
||||
gpath.insert(self.obj, ["foo", "foo"], 17)
|
||||
|
||||
def test_update(self):
|
||||
@ -75,13 +75,13 @@ class TestGpath(unittest.TestCase):
|
||||
|
||||
def test_update_strict(self):
|
||||
"""update should be strict"""
|
||||
with self.assertRaisesRegexp(Exception, r'non-existent'):
|
||||
with self.assertRaisesRegex(Exception, r'non-existent'):
|
||||
gpath.update(self.obj, ["bar", 4], 17)
|
||||
with self.assertRaisesRegexp(Exception, r'not an array'):
|
||||
with self.assertRaisesRegex(Exception, r'not an array'):
|
||||
gpath.update(self.obj, ["foo"], 17)
|
||||
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
|
||||
with self.assertRaisesRegex(Exception, r'invalid.*index'):
|
||||
gpath.update(self.obj, ["foo", -1], 17)
|
||||
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
|
||||
with self.assertRaisesRegex(Exception, r'invalid.*index'):
|
||||
gpath.update(self.obj, ["foo", None], 17)
|
||||
|
||||
def test_remove(self):
|
||||
@ -96,13 +96,13 @@ class TestGpath(unittest.TestCase):
|
||||
|
||||
def test_remove_strict(self):
|
||||
"""remove should be strict"""
|
||||
with self.assertRaisesRegexp(Exception, r'non-existent'):
|
||||
with self.assertRaisesRegex(Exception, r'non-existent'):
|
||||
gpath.remove(self.obj, ["bar", 4])
|
||||
with self.assertRaisesRegexp(Exception, r'not an array'):
|
||||
with self.assertRaisesRegex(Exception, r'not an array'):
|
||||
gpath.remove(self.obj, ["foo"])
|
||||
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
|
||||
with self.assertRaisesRegex(Exception, r'invalid.*index'):
|
||||
gpath.remove(self.obj, ["foo", -1])
|
||||
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
|
||||
with self.assertRaisesRegex(Exception, r'invalid.*index'):
|
||||
gpath.remove(self.obj, ["foo", None])
|
||||
|
||||
def test_glob(self):
|
||||
@ -112,7 +112,7 @@ class TestGpath(unittest.TestCase):
|
||||
self.assertEqual(gpath.place(self.obj, ["foo", "*", "bar"], 17), 3)
|
||||
self.assertEqual(self.obj["foo"], [{"bar": 17}, {"bar": 17}, {"baz": 3, "bar": 17}])
|
||||
|
||||
with self.assertRaisesRegexp(Exception, r'non-existent object at \/foo\/\*\/bad'):
|
||||
with self.assertRaisesRegex(Exception, r'non-existent object at \/foo\/\*\/bad'):
|
||||
gpath.place(self.obj, ["foo", "*", "bad", "test"], 10)
|
||||
|
||||
self.assertEqual(gpath.update(self.obj, ["foo", "*"], "hello"), 3)
|
||||
@ -120,9 +120,9 @@ class TestGpath(unittest.TestCase):
|
||||
|
||||
def test_glob_strict_wildcard(self):
|
||||
"""should only support tail wildcard for updates"""
|
||||
with self.assertRaisesRegexp(Exception, r'invalid array index'):
|
||||
with self.assertRaisesRegex(Exception, r'invalid array index'):
|
||||
gpath.remove(self.obj, ["foo", "*"])
|
||||
with self.assertRaisesRegexp(Exception, r'invalid array index'):
|
||||
with self.assertRaisesRegex(Exception, r'invalid array index'):
|
||||
gpath.insert(self.obj, ["foo", "*"], 1)
|
||||
|
||||
def test_glob_wildcard_keys(self):
|
||||
@ -132,7 +132,7 @@ class TestGpath(unittest.TestCase):
|
||||
self.assertEqual(gpath.place(self.obj, ["foo", 0, "*"], 17), 1)
|
||||
self.assertEqual(self.obj["foo"], [{"bar": 1, '*': 17}, {"bar": 2}, {"baz": 3}])
|
||||
|
||||
with self.assertRaisesRegexp(Exception, r'non-existent'):
|
||||
with self.assertRaisesRegex(Exception, r'non-existent'):
|
||||
gpath.place(self.obj, ["*", 0, "bar"], 17)
|
||||
|
||||
def test_glob_nested(self):
|
||||
|
||||
@ -13,7 +13,7 @@ class TestImportActions(test_engine.EngineTestCase):
|
||||
{'id': 'Zip', 'type': 'Int'}]])
|
||||
self.apply_user_action(['BulkAddRecord', 'Source', [1, 2], {'Name': ['John', 'Alison'],
|
||||
'City': ['New York', 'Boston'],
|
||||
'Zip': [03011, 07003]}])
|
||||
'Zip': [3011, 7003]}])
|
||||
self.assertTableData('_grist_Tables_column', cols="subset", data=[
|
||||
["id", "colId", "type", "isFormula", "formula"],
|
||||
[1, "manualSort", "ManualSortPos", False, ""],
|
||||
@ -79,8 +79,8 @@ class TestImportActions(test_engine.EngineTestCase):
|
||||
|
||||
self.assertTableData('Source', cols="all", data=[
|
||||
["id", "Name", "City", "Zip", "gristHelper_Import_Name", "gristHelper_Import_City", "manualSort"],
|
||||
[1, "John", "New York", 03011, "John", "New York", 1.0],
|
||||
[2, "Alison", "Boston", 07003, "Alison", "Boston", 2.0],
|
||||
[1, "John", "New York", 3011, "John", "New York", 1.0],
|
||||
[2, "Alison", "Boston", 7003, "Alison", "Boston", 2.0],
|
||||
])
|
||||
|
||||
self.assertPartialData("_grist_Views_section", ["id", "tableRef", 'fields'], [
|
||||
@ -107,8 +107,8 @@ class TestImportActions(test_engine.EngineTestCase):
|
||||
|
||||
self.assertTableData('Source', cols="all", data=[
|
||||
["id", "Name", "City", "Zip", "gristHelper_Import_State", "manualSort"],
|
||||
[1, "John", "New York", 03011, "", 1.0],
|
||||
[2, "Alison", "Boston", 07003, "", 2.0],
|
||||
[1, "John", "New York", 3011, "", 1.0],
|
||||
[2, "Alison", "Boston", 7003, "", 2.0],
|
||||
])
|
||||
self.assertPartialData("_grist_Views_section", ["id", "tableRef", 'fields'], [
|
||||
[1, 1, [1, 2, 3]],
|
||||
@ -168,8 +168,8 @@ class TestImportActions(test_engine.EngineTestCase):
|
||||
|
||||
self.assertTableData('Source', cols="all", data=[
|
||||
["id", "Name", "City", "Zip", "gristHelper_Import_Name", "gristHelper_Import_City", "gristHelper_Import_Zip", "manualSort"],
|
||||
[1, "John", "New York", 03011, "John", "New York", 03011, 1.0],
|
||||
[2, "Alison", "Boston", 07003, "Alison", "Boston", 07003, 2.0],
|
||||
[1, "John", "New York", 3011, "John", "New York", 3011, 1.0],
|
||||
[2, "Alison", "Boston", 7003, "Alison", "Boston", 7003, 2.0],
|
||||
])
|
||||
self.assertPartialData("_grist_Views_section", ["id", "tableRef", 'fields'], [
|
||||
[1, 1, [1, 2, 3]],
|
||||
|
||||
@ -3,6 +3,10 @@ import string
|
||||
import timeit
|
||||
import unittest
|
||||
from collections import Hashable
|
||||
|
||||
import six
|
||||
from six.moves import xrange
|
||||
|
||||
import match_counter
|
||||
from testutil import repeat_until_passes
|
||||
|
||||
@ -22,7 +26,7 @@ class MatchCounterOther(object):
|
||||
pass
|
||||
|
||||
matches = 0
|
||||
for v, n in self.sample_counts.iteritems():
|
||||
for v, n in six.iteritems(self.sample_counts):
|
||||
if n > 0:
|
||||
matches += 1
|
||||
self.sample_counts[v] = 0
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
import unittest
|
||||
|
||||
import six
|
||||
|
||||
import actions
|
||||
import schema
|
||||
import table_data_set
|
||||
@ -23,7 +25,7 @@ class TestMigrations(unittest.TestCase):
|
||||
|
||||
# Figure out the missing actions.
|
||||
doc_actions = []
|
||||
for table_id in sorted(current_schema.viewkeys() | migrated_schema.viewkeys()):
|
||||
for table_id in sorted(six.viewkeys(current_schema) | six.viewkeys(migrated_schema)):
|
||||
if table_id not in migrated_schema:
|
||||
doc_actions.append(actions.AddTable(table_id, current_schema[table_id].values()))
|
||||
elif table_id not in current_schema:
|
||||
@ -31,7 +33,7 @@ class TestMigrations(unittest.TestCase):
|
||||
else:
|
||||
current_cols = current_schema[table_id]
|
||||
migrated_cols = migrated_schema[table_id]
|
||||
for col_id in sorted(current_cols.viewkeys() | migrated_cols.viewkeys()):
|
||||
for col_id in sorted(six.viewkeys(current_cols) | six.viewkeys(migrated_cols)):
|
||||
if col_id not in migrated_cols:
|
||||
doc_actions.append(actions.AddColumn(table_id, col_id, current_cols[col_id]))
|
||||
elif col_id not in current_cols:
|
||||
@ -39,7 +41,7 @@ class TestMigrations(unittest.TestCase):
|
||||
else:
|
||||
current_info = current_cols[col_id]
|
||||
migrated_info = migrated_cols[col_id]
|
||||
delta = {k: v for k, v in current_info.iteritems() if v != migrated_info.get(k)}
|
||||
delta = {k: v for k, v in six.iteritems(current_info) if v != migrated_info.get(k)}
|
||||
if delta:
|
||||
doc_actions.append(actions.ModifyColumn(table_id, col_id, delta))
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@ class TestMoment(unittest.TestCase):
|
||||
[datetime(1979, 10, 28, 6, 0, 0), 309938400000, "EST", 300, 1, 0],
|
||||
# - 2037 -
|
||||
[datetime(2037, 3, 8, 6, 59, 59), 2120108399000, "EST", 300, 1, 59],
|
||||
[datetime(2037, 03, 8, 7, 0, 0), 2120108400000, "EDT", 240, 3, 0],
|
||||
[datetime(2037, 3, 8, 7, 0, 0), 2120108400000, "EDT", 240, 3, 0],
|
||||
[datetime(2037, 11, 1, 5, 59, 59), 2140667999000, "EDT", 240, 1, 59]
|
||||
]
|
||||
new_york_errors = [
|
||||
@ -255,7 +255,7 @@ class TestMoment(unittest.TestCase):
|
||||
|
||||
def test_dt_to_ds(self):
|
||||
# Verify that dt_to_ts works for both naive and aware datetime objects.
|
||||
value_dt = datetime(2015, 03, 14, 0, 0) # In UTC
|
||||
value_dt = datetime(2015, 3, 14, 0, 0) # In UTC
|
||||
value_sec = 1426291200
|
||||
tzla = moment.get_zone('America/Los_Angeles')
|
||||
def format_utc(ts):
|
||||
@ -287,7 +287,7 @@ class TestMoment(unittest.TestCase):
|
||||
self.assertEqual(value_dt_aware.strftime(fmt), '2015-02-13 20:00:00 EST')
|
||||
|
||||
def test_date_to_ts(self):
|
||||
d = date(2015, 03, 14)
|
||||
d = date(2015, 3, 14)
|
||||
tzla = moment.get_zone('America/Los_Angeles')
|
||||
def format_utc(ts):
|
||||
return moment.ts_to_dt(ts, moment.get_zone('UTC')).strftime(fmt)
|
||||
|
||||
@ -7,6 +7,7 @@ import objtypes
|
||||
import testsamples
|
||||
import testutil
|
||||
import test_engine
|
||||
from objtypes import RecordStub
|
||||
|
||||
log = logger.Logger(__name__, logger.INFO)
|
||||
|
||||
@ -18,84 +19,84 @@ class TestRecordFunc(test_engine.EngineTestCase):
|
||||
|
||||
def test_record_self(self):
|
||||
self.load_sample(testsamples.sample_students)
|
||||
self.add_column("Schools", "Foo", formula='repr(RECORD(rec))')
|
||||
self.add_column("Schools", "Foo", formula='RECORD(rec)')
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"],
|
||||
[2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"],
|
||||
[3, "{'address': Address[13], 'id': 3, 'name': 'Yale'}"],
|
||||
[4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"],
|
||||
[1, {'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}],
|
||||
[2, {'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}],
|
||||
[3, {'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}],
|
||||
[4, {'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}],
|
||||
])
|
||||
|
||||
# A change to data is reflected
|
||||
self.update_record("Schools", 3, name="UConn")
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"],
|
||||
[2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"],
|
||||
[3, "{'address': Address[13], 'id': 3, 'name': 'UConn'}"],
|
||||
[4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"],
|
||||
[1, {'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}],
|
||||
[2, {'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}],
|
||||
[3, {'address': RecordStub('Address', 13), 'id': 3, 'name': 'UConn'}],
|
||||
[4, {'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}],
|
||||
])
|
||||
|
||||
# A column addition is reflected
|
||||
self.add_column("Schools", "Bar", formula='len($name)')
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': Address[11], 'Bar': 8, 'id': 1, 'name': 'Columbia'}"],
|
||||
[2, "{'address': Address[12], 'Bar': 8, 'id': 2, 'name': 'Columbia'}"],
|
||||
[3, "{'address': Address[13], 'Bar': 5, 'id': 3, 'name': 'UConn'}"],
|
||||
[4, "{'address': Address[14], 'Bar': 4, 'id': 4, 'name': 'Yale'}"],
|
||||
[1, {'address': RecordStub('Address', 11), 'Bar': 8, 'id': 1, 'name': 'Columbia'}],
|
||||
[2, {'address': RecordStub('Address', 12), 'Bar': 8, 'id': 2, 'name': 'Columbia'}],
|
||||
[3, {'address': RecordStub('Address', 13), 'Bar': 5, 'id': 3, 'name': 'UConn'}],
|
||||
[4, {'address': RecordStub('Address', 14), 'Bar': 4, 'id': 4, 'name': 'Yale'}],
|
||||
])
|
||||
|
||||
def test_reference(self):
|
||||
self.load_sample(testsamples.sample_students)
|
||||
self.add_column("Schools", "Foo", formula='repr(RECORD($address))')
|
||||
self.add_column("Schools", "Foo", formula='RECORD($address)')
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'city': 'New York', 'id': 11}"],
|
||||
[2, "{'city': 'Colombia', 'id': 12}"],
|
||||
[3, "{'city': 'New Haven', 'id': 13}"],
|
||||
[4, "{'city': 'West Haven', 'id': 14}"],
|
||||
[1, {'city': 'New York', 'id': 11}],
|
||||
[2, {'city': 'Colombia', 'id': 12}],
|
||||
[3, {'city': 'New Haven', 'id': 13}],
|
||||
[4, {'city': 'West Haven', 'id': 14}],
|
||||
])
|
||||
|
||||
# A change to referenced data is still reflected; try a different kind of change here
|
||||
self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'ciudad': 'New York', 'id': 11}"],
|
||||
[2, "{'ciudad': 'Colombia', 'id': 12}"],
|
||||
[3, "{'ciudad': 'New Haven', 'id': 13}"],
|
||||
[4, "{'ciudad': 'West Haven', 'id': 14}"],
|
||||
[1, {'ciudad': 'New York', 'id': 11}],
|
||||
[2, {'ciudad': 'Colombia', 'id': 12}],
|
||||
[3, {'ciudad': 'New Haven', 'id': 13}],
|
||||
[4, {'ciudad': 'West Haven', 'id': 14}],
|
||||
])
|
||||
|
||||
def test_record_expand_refs(self):
|
||||
self.load_sample(testsamples.sample_students)
|
||||
self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))')
|
||||
self.add_column("Schools", "Foo", formula='RECORD(rec, expand_refs=1)')
|
||||
self.add_column("Address", "student", type="Ref:Students")
|
||||
self.update_record("Address", 12, student=6)
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': {'city': 'New York', 'id': 11, 'student': Students[0]}," +
|
||||
" 'id': 1, 'name': 'Columbia'}"],
|
||||
[2, "{'address': {'city': 'Colombia', 'id': 12, 'student': Students[6]}," +
|
||||
" 'id': 2, 'name': 'Columbia'}"],
|
||||
[3, "{'address': {'city': 'New Haven', 'id': 13, 'student': Students[0]}," +
|
||||
" 'id': 3, 'name': 'Yale'}"],
|
||||
[4, "{'address': {'city': 'West Haven', 'id': 14, 'student': Students[0]}," +
|
||||
" 'id': 4, 'name': 'Yale'}"],
|
||||
[1, {'address': {'city': 'New York', 'id': 11, 'student': RecordStub("Students", 0)},
|
||||
'id': 1, 'name': 'Columbia'}],
|
||||
[2, {'address': {'city': 'Colombia', 'id': 12, 'student': RecordStub("Students", 6)},
|
||||
'id': 2, 'name': 'Columbia'}],
|
||||
[3, {'address': {'city': 'New Haven', 'id': 13, 'student': RecordStub("Students", 0)},
|
||||
'id': 3, 'name': 'Yale'}],
|
||||
[4, {'address': {'city': 'West Haven', 'id': 14, 'student': RecordStub("Students", 0)},
|
||||
'id': 4, 'name': 'Yale'}],
|
||||
])
|
||||
|
||||
self.modify_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=2))')
|
||||
self.modify_column("Schools", "Foo", formula='RECORD(rec, expand_refs=2)')
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': {'city': 'New York', 'id': 11, 'student': None}," +
|
||||
" 'id': 1, 'name': 'Columbia'}"],
|
||||
[2, "{'address': {'city': 'Colombia', 'id': 12, " +
|
||||
"'student': {'firstName': 'Gerald', 'schoolName': 'Yale', 'lastName': 'Ford', " +
|
||||
"'schoolCities': 'New Haven:West Haven', 'schoolIds': '3:4', 'id': 6}}," +
|
||||
" 'id': 2, 'name': 'Columbia'}"],
|
||||
[3, "{'address': {'city': 'New Haven', 'id': 13, 'student': None}," +
|
||||
" 'id': 3, 'name': 'Yale'}"],
|
||||
[4, "{'address': {'city': 'West Haven', 'id': 14, 'student': None}," +
|
||||
" 'id': 4, 'name': 'Yale'}"],
|
||||
[1, {'address': {'city': 'New York', 'id': 11, 'student': None},
|
||||
'id': 1, 'name': 'Columbia'}],
|
||||
[2, {'address': {'city': 'Colombia', 'id': 12,
|
||||
'student': {'firstName': 'Gerald', 'schoolName': 'Yale', 'lastName': 'Ford',
|
||||
'schoolCities': 'New Haven:West Haven', 'schoolIds': '3:4', 'id': 6}},
|
||||
'id': 2, 'name': 'Columbia'}],
|
||||
[3, {'address': {'city': 'New Haven', 'id': 13, 'student': None},
|
||||
'id': 3, 'name': 'Yale'}],
|
||||
[4, {'address': {'city': 'West Haven', 'id': 14, 'student': None},
|
||||
'id': 4, 'name': 'Yale'}],
|
||||
])
|
||||
|
||||
def test_record_date_options(self):
|
||||
self.load_sample(testsamples.sample_students)
|
||||
self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))')
|
||||
self.add_column("Schools", "Foo", formula='RECORD(rec, expand_refs=1)')
|
||||
self.add_column("Address", "DT", type='DateTime')
|
||||
self.add_column("Address", "D", type='Date', formula="$DT and $DT.date()")
|
||||
self.update_records("Address", ['id', 'DT'], [
|
||||
@ -106,61 +107,57 @@ class TestRecordFunc(test_engine.EngineTestCase):
|
||||
d1 = datetime.datetime(2020, 9, 13, 8, 26, 40, tzinfo=moment.tzinfo('America/New_York'))
|
||||
d2 = datetime.datetime(2017, 7, 13, 22, 40, tzinfo=moment.tzinfo('America/New_York'))
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': {'city': 'New York', 'DT': %s, 'id': 11, 'D': %s}, " %
|
||||
(repr(d1), repr(d1.date())) +
|
||||
"'id': 1, 'name': 'Columbia'}"],
|
||||
[2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
|
||||
"'id': 2, 'name': 'Columbia'}"],
|
||||
[3, "{'address': {'city': 'New Haven', 'DT': %s, 'id': 13, 'D': %s}, " %
|
||||
(repr(d2), repr(d2.date())) +
|
||||
"'id': 3, 'name': 'Yale'}"],
|
||||
[4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
|
||||
"'id': 4, 'name': 'Yale'}"],
|
||||
[1, {'address': {'city': 'New York', 'DT': d1, 'id': 11, 'D': d1.date()},
|
||||
'id': 1, 'name': 'Columbia'}],
|
||||
[2, {'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None},
|
||||
'id': 2, 'name': 'Columbia'}],
|
||||
[3, {'address': {'city': 'New Haven', 'DT': d2, 'id': 13, 'D': d2.date()},
|
||||
'id': 3, 'name': 'Yale'}],
|
||||
[4, {'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None},
|
||||
'id': 4, 'name': 'Yale'}],
|
||||
])
|
||||
|
||||
self.modify_column("Schools", "Foo",
|
||||
formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=True))')
|
||||
formula='RECORD(rec, expand_refs=1, dates_as_iso=True)')
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " %
|
||||
(d1.isoformat(), d1.date().isoformat()) +
|
||||
"'id': 1, 'name': 'Columbia'}"],
|
||||
[2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
|
||||
"'id': 2, 'name': 'Columbia'}"],
|
||||
[3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " %
|
||||
(d2.isoformat(), d2.date().isoformat()) +
|
||||
"'id': 3, 'name': 'Yale'}"],
|
||||
[4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
|
||||
"'id': 4, 'name': 'Yale'}"],
|
||||
[1, {'address': {'city': 'New York', 'DT': d1.isoformat(), 'id': 11, 'D': d1.date().isoformat()},
|
||||
'id': 1, 'name': 'Columbia'}],
|
||||
[2, {'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None},
|
||||
'id': 2, 'name': 'Columbia'}],
|
||||
[3, {'address': {'city': 'New Haven', 'DT': d2.isoformat(), 'id': 13, 'D': d2.date().isoformat()},
|
||||
'id': 3, 'name': 'Yale'}],
|
||||
[4, {'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None},
|
||||
'id': 4, 'name': 'Yale'}],
|
||||
])
|
||||
|
||||
def test_record_set(self):
|
||||
self.load_sample(testsamples.sample_students)
|
||||
self.add_column("Students", "schools", formula='Schools.lookupRecords(name=$schoolName)')
|
||||
self.add_column("Students", "Foo", formula='repr(RECORD($schools))')
|
||||
self.add_column("Students", "Foo", formula='RECORD($schools)')
|
||||
self.assertPartialData("Students", ["id", "Foo"], [
|
||||
[1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," +
|
||||
" {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
|
||||
[2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
|
||||
" {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
|
||||
[3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," +
|
||||
" {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
|
||||
[4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
|
||||
" {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
|
||||
[5, "[]"],
|
||||
[6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
|
||||
" {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
|
||||
[1, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'},
|
||||
{'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}]],
|
||||
[2, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'},
|
||||
{'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}]],
|
||||
[3, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'},
|
||||
{'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}]],
|
||||
[4, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'},
|
||||
{'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}]],
|
||||
[5, []],
|
||||
[6, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'},
|
||||
{'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}]],
|
||||
])
|
||||
|
||||
# Try a field with filtered lookupRecords result, as an iterable.
|
||||
self.modify_column("Students", "Foo",
|
||||
formula='repr(RECORD(s for s in $schools if s.address.city.startswith("New")))')
|
||||
formula='RECORD(s for s in $schools if s.address.city.startswith("New"))')
|
||||
self.assertPartialData("Students", ["id", "Foo"], [
|
||||
[1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"],
|
||||
[2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
|
||||
[3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"],
|
||||
[4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
|
||||
[5, "[]"],
|
||||
[6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
|
||||
[1, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}]],
|
||||
[2, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}]],
|
||||
[3, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}]],
|
||||
[4, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}]],
|
||||
[5, []],
|
||||
[6, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}]],
|
||||
])
|
||||
|
||||
def test_record_bad_calls(self):
|
||||
@ -172,25 +169,25 @@ class TestRecordFunc(test_engine.EngineTestCase):
|
||||
[3, objtypes.RaisedException(ValueError())],
|
||||
[4, objtypes.RaisedException(ValueError())],
|
||||
])
|
||||
self.modify_column("Schools", "Foo", formula='repr(RECORD([rec] if $id == 2 else $id))')
|
||||
self.modify_column("Schools", "Foo", formula='repr(sorted(RECORD(rec if $id == 2 else $id).items()))')
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, objtypes.RaisedException(ValueError())],
|
||||
[2, "[{'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
|
||||
[2, "[('address', Address[12]), ('id', 2), ('name', 'Columbia')]"],
|
||||
[3, objtypes.RaisedException(ValueError())],
|
||||
[4, objtypes.RaisedException(ValueError())],
|
||||
])
|
||||
self.assertEqual(self.engine.get_formula_error('Schools', 'Foo', 1).error.message,
|
||||
self.assertEqual(str(self.engine.get_formula_error('Schools', 'Foo', 1).error),
|
||||
'RECORD() requires a Record or an iterable of Records')
|
||||
|
||||
def test_record_error_cells(self):
|
||||
self.load_sample(testsamples.sample_students)
|
||||
self.add_column("Schools", "Foo", formula='repr(RECORD($address))')
|
||||
self.add_column("Address", "Bar", formula='$id/($id%2)')
|
||||
self.add_column("Schools", "Foo", formula='RECORD($address)')
|
||||
self.add_column("Address", "Bar", formula='$id//($id%2)')
|
||||
self.assertPartialData("Schools", ["id", "Foo"], [
|
||||
[1, "{'city': 'New York', 'Bar': 11, 'id': 11}"],
|
||||
[2, "{'city': 'Colombia', 'Bar': None, 'id': 12, " +
|
||||
"'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"],
|
||||
[3, "{'city': 'New Haven', 'Bar': 13, 'id': 13}"],
|
||||
[4, "{'city': 'West Haven', 'Bar': None, 'id': 14, " +
|
||||
"'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"],
|
||||
[1, {'city': 'New York', 'Bar': 11, 'id': 11}],
|
||||
[2, {'city': 'Colombia', 'Bar': None, 'id': 12,
|
||||
'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}],
|
||||
[3, {'city': 'New Haven', 'Bar': 13, 'id': 13}],
|
||||
[4, {'city': 'West Haven', 'Bar': None, 'id': 14,
|
||||
'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}],
|
||||
])
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
import relabeling
|
||||
|
||||
from sortedcontainers import SortedListWithKey
|
||||
from itertools import izip
|
||||
from six.moves import zip as izip, xrange
|
||||
|
||||
import unittest
|
||||
import sys
|
||||
|
||||
@ -264,7 +265,7 @@ class TestRelabeling(unittest.TestCase):
|
||||
slist.insert_items([(i, float('-inf')), (-i, float('inf'))])
|
||||
|
||||
self.assertEqual(slist.get_values(),
|
||||
rev_range(2000) + [v for v,k in initial] + range(0, -2000, -1))
|
||||
rev_range(2000) + [v for v,k in initial] + list(xrange(0, -2000, -1)))
|
||||
#print slist.num_update_events, slist.num_updated_keys
|
||||
self.assertLess(slist.avg_updated_keys(), 3)
|
||||
self.assertLess(slist.num_update_events, 80)
|
||||
@ -276,7 +277,7 @@ class TestRelabeling(unittest.TestCase):
|
||||
slist.insert_items([(i, ins_item.key)])
|
||||
|
||||
# Check the end result
|
||||
self.assertEqual(slist.get_values(), ['a', 'b'] + range(1000) + ['c', 'd'])
|
||||
self.assertEqual(slist.get_values(), ['a', 'b'] + list(xrange(1000)) + ['c', 'd'])
|
||||
self.assertAlmostEqual(slist.avg_updated_keys(), 3.5, delta=1)
|
||||
self.assertLess(slist.num_update_events, 40)
|
||||
|
||||
@ -299,7 +300,7 @@ class TestRelabeling(unittest.TestCase):
|
||||
ins_item = slist.find_value('c')
|
||||
for i in xrange(1000):
|
||||
slist.insert_items([(i, ins_item.key)], prepare_inserts=r.prepare_inserts_dumb)
|
||||
self.assertEqual(slist.get_values(), ['a', 'b'] + range(1000) + ['c', 'd'])
|
||||
self.assertEqual(slist.get_values(), ['a', 'b'] + list(xrange(1000)) + ['c', 'd'])
|
||||
self.assertGreater(slist.avg_updated_keys(), 8)
|
||||
|
||||
def test_renumber_right_dumb(self):
|
||||
@ -334,7 +335,7 @@ class TestRelabeling(unittest.TestCase):
|
||||
# aL1, al0, al1, a, ar1, ar0, aR1, ...
|
||||
# aL1, al0, aL2, al1, al2, a, ar2, ar1, aR2, ar0, aR1, ...
|
||||
def left_half(val):
|
||||
half = range(2*N - 1)
|
||||
half = list(xrange(2*N - 1))
|
||||
half[0::2] = ['%sL%d' % (val, i) for i in xrange(1, N + 1)]
|
||||
half[1::2] = ['%sl%d' % (val, i) for i in xrange(0, N - 1)]
|
||||
half[-1] = '%sl%d' % (val, N - 1)
|
||||
@ -355,7 +356,7 @@ class TestRelabeling(unittest.TestCase):
|
||||
|
||||
|
||||
def rev_range(n):
|
||||
return list(reversed(range(n)))
|
||||
return list(reversed(list(xrange(n))))
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@ -31,7 +31,7 @@ class TestSummary2(test_engine.EngineTestCase):
|
||||
self.apply_user_action(["CreateViewSection", 1, 0, "record", []])
|
||||
|
||||
# Check that we cannot add a non-formula column.
|
||||
with self.assertRaisesRegexp(ValueError, r'non-formula column'):
|
||||
with self.assertRaisesRegex(ValueError, r'non-formula column'):
|
||||
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "average",
|
||||
{"type": "Text", "isFormula": False}])
|
||||
|
||||
@ -171,7 +171,7 @@ class TestSummary2(test_engine.EngineTestCase):
|
||||
|
||||
# Check that we cannot rename a summary group-by column. (Perhaps it's better to raise an
|
||||
# exception, but currently we translate the invalid request to a no-op.)
|
||||
with self.assertRaisesRegexp(ValueError, r'Cannot modify .* group-by'):
|
||||
with self.assertRaisesRegex(ValueError, r'Cannot modify .* group-by'):
|
||||
self.apply_user_action(["RenameColumn", "GristSummary_7_Address", "state", "s"])
|
||||
|
||||
# Verify all data. We'll repeat this after renamings to make sure there are no errors.
|
||||
@ -353,46 +353,46 @@ class TestSummary2(test_engine.EngineTestCase):
|
||||
])
|
||||
|
||||
# (1) no adding/removing/renaming non-formula columns.
|
||||
with self.assertRaisesRegexp(ValueError, r'non-formula column'):
|
||||
with self.assertRaisesRegex(ValueError, r'non-formula column'):
|
||||
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo",
|
||||
{"type": "Numeric", "isFormula": False}])
|
||||
|
||||
with self.assertRaisesRegexp(ValueError, r'group-by column'):
|
||||
with self.assertRaisesRegex(ValueError, r'group-by column'):
|
||||
self.apply_user_action(["RemoveColumn", "GristSummary_7_Address", "state"])
|
||||
|
||||
with self.assertRaisesRegexp(ValueError, r'Cannot modify .* group-by'):
|
||||
with self.assertRaisesRegex(ValueError, r'Cannot modify .* group-by'):
|
||||
self.apply_user_action(["RenameColumn", "GristSummary_7_Address", "state", "st"])
|
||||
|
||||
# (2) no converting between formula/non-formula
|
||||
with self.assertRaisesRegexp(ValueError, r'Cannot change .* formula and data'):
|
||||
with self.assertRaisesRegex(ValueError, r'Cannot change .* formula and data'):
|
||||
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "amount",
|
||||
{"isFormula": False}])
|
||||
|
||||
with self.assertRaisesRegexp(ValueError, r'Cannot change .* formula and data'):
|
||||
with self.assertRaisesRegex(ValueError, r'Cannot change .* formula and data'):
|
||||
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "state",
|
||||
{"isFormula": True}])
|
||||
|
||||
# (3) no editing values in non-formula columns
|
||||
with self.assertRaisesRegexp(ValueError, r'Cannot enter data .* group-by'):
|
||||
with self.assertRaisesRegex(ValueError, r'Cannot enter data .* group-by'):
|
||||
self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 6, {"state": "ny"}])
|
||||
|
||||
# (4) no removing rows (this is questionable b/c empty rows might be OK to remove)
|
||||
with self.assertRaisesRegexp(ValueError, r'Cannot remove record .* summary'):
|
||||
with self.assertRaisesRegex(ValueError, r'Cannot remove record .* summary'):
|
||||
self.apply_user_action(["RemoveRecord", "GristSummary_7_Address", 6])
|
||||
|
||||
# (5) no renaming summary tables.
|
||||
with self.assertRaisesRegexp(ValueError, r'cannot rename .* summary'):
|
||||
with self.assertRaisesRegex(ValueError, r'cannot rename .* summary'):
|
||||
self.apply_user_action(["RenameTable", "GristSummary_7_Address", "GristSummary_hello"])
|
||||
|
||||
# Check that we can add an empty column, then set a formula for it.
|
||||
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo", {}])
|
||||
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "foo", {"formula": "1+1"}])
|
||||
with self.assertRaisesRegexp(ValueError, "Can't save .* to formula"):
|
||||
with self.assertRaisesRegex(ValueError, "Can't save .* to formula"):
|
||||
self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 1, {"foo": "hello"}])
|
||||
|
||||
# But we cannot add an empty column, then add a value to it.
|
||||
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo2", {}])
|
||||
with self.assertRaisesRegexp(ValueError, r'Cannot change .* between formula and data'):
|
||||
with self.assertRaisesRegex(ValueError, r'Cannot change .* between formula and data'):
|
||||
self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 1, {"foo2": "hello"}])
|
||||
|
||||
self.assertTableData('GristSummary_7_Address', cols="all", data=[
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
import six
|
||||
|
||||
import actions
|
||||
import schema
|
||||
import table_data_set
|
||||
@ -37,7 +39,7 @@ class TestTableDataSet(unittest.TestCase):
|
||||
if a.table_id not in self._table_data_set.all_tables:
|
||||
self._table_data_set.apply_doc_action(a)
|
||||
|
||||
for a in sample["SCHEMA"].itervalues():
|
||||
for a in six.itervalues(sample["SCHEMA"]):
|
||||
self._table_data_set.BulkAddRecord(*a)
|
||||
|
||||
# Create AddTable actions for each table described in the metadata.
|
||||
@ -61,11 +63,11 @@ class TestTableDataSet(unittest.TestCase):
|
||||
})
|
||||
|
||||
# Sort the columns in the schema according to the parentPos field from the column records.
|
||||
for action in add_tables.itervalues():
|
||||
for action in six.itervalues(add_tables):
|
||||
action.columns.sort(key=lambda r: r["parentPos"])
|
||||
self._table_data_set.AddTable(*action)
|
||||
|
||||
for a in sample["DATA"].itervalues():
|
||||
for a in six.itervalues(sample["DATA"]):
|
||||
self._table_data_set.ReplaceTableData(*a)
|
||||
|
||||
|
||||
@ -92,11 +94,11 @@ class TestTableDataSet(unittest.TestCase):
|
||||
if "USE_SAMPLE" in data:
|
||||
expected_data = self.samples[data.pop("USE_SAMPLE")]["DATA"].copy()
|
||||
expected_data.update({t: testutil.table_data_from_rows(t, tdata[0], tdata[1:])
|
||||
for (t, tdata) in data.iteritems()})
|
||||
for (t, tdata) in six.iteritems(data)})
|
||||
self._verify_data(expected_data)
|
||||
else:
|
||||
raise ValueError("Unrecognized step %s in test script" % step)
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
new_args0 = "LINE %s: %s" % (line, e.args[0])
|
||||
e.args = (new_args0,) + e.args[1:]
|
||||
raise
|
||||
@ -117,7 +119,7 @@ class TestTableDataSet(unittest.TestCase):
|
||||
|
||||
def _verify_data(self, expected_data, ignore_formulas=False):
|
||||
observed_data = {t: self._prep_data(*data)
|
||||
for t, data in self._table_data_set.all_tables.iteritems()
|
||||
for t, data in six.iteritems(self._table_data_set.all_tables)
|
||||
if not t.startswith("_grist_")}
|
||||
if ignore_formulas:
|
||||
observed_data = self._strip_formulas(observed_data)
|
||||
@ -125,7 +127,7 @@ class TestTableDataSet(unittest.TestCase):
|
||||
|
||||
if observed_data != expected_data:
|
||||
lines = []
|
||||
for table in sorted(observed_data.viewkeys() | expected_data.viewkeys()):
|
||||
for table in sorted(six.viewkeys(observed_data) | six.viewkeys(expected_data)):
|
||||
if table not in expected_data:
|
||||
lines.append("*** Table %s observed but not expected\n" % table)
|
||||
elif table not in observed_data:
|
||||
@ -141,11 +143,11 @@ class TestTableDataSet(unittest.TestCase):
|
||||
self.fail("\n" + "".join(lines))
|
||||
|
||||
def _strip_formulas(self, all_data):
|
||||
return {t: self._strip_formulas_table(*data) for t, data in all_data.iteritems()}
|
||||
return {t: self._strip_formulas_table(*data) for t, data in six.iteritems(all_data)}
|
||||
|
||||
def _strip_formulas_table(self, table_id, row_ids, columns):
|
||||
return actions.TableData(table_id, row_ids, {
|
||||
col_id: col for col_id, col in columns.iteritems()
|
||||
col_id: col for col_id, col in six.iteritems(columns)
|
||||
if not self._table_data_set.get_col_info(table_id, col_id)["isFormula"]
|
||||
})
|
||||
|
||||
@ -155,7 +157,7 @@ class TestTableDataSet(unittest.TestCase):
|
||||
return [v for r, v in sorted(zip(row_ids, col))]
|
||||
|
||||
sorted_data = actions.TableData(table_id, sorted(row_ids),
|
||||
{c: sort(col) for c, col in columns.iteritems()})
|
||||
{c: sort(col) for c, col in six.iteritems(columns)})
|
||||
return actions.encode_objects(testutil.replace_nans(sorted_data))
|
||||
|
||||
@classmethod
|
||||
|
||||
@ -9,7 +9,7 @@ class TestTextBuilder(unittest.TestCase):
|
||||
def test_validate_patch(self):
|
||||
text = "To be or not to be"
|
||||
patch = make_patch(text, 3, 8, "SEE OR")
|
||||
self.assertEquals(textbuilder.validate_patch(text, patch), None)
|
||||
self.assertEqual(textbuilder.validate_patch(text, patch), None)
|
||||
with self.assertRaises(ValueError):
|
||||
textbuilder.validate_patch('X' + text, patch)
|
||||
|
||||
@ -19,15 +19,15 @@ class TestTextBuilder(unittest.TestCase):
|
||||
patches = make_regexp_patches(t1.get_text(), re.compile(r'be|to', re.I),
|
||||
lambda m: (m.group() + m.group()).upper())
|
||||
t2 = textbuilder.Replacer(t1, patches)
|
||||
self.assertEquals(t2.get_text(), "TOTO BEBE or not\n TOTO BEBE?\n")
|
||||
self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 0, 4, "xxx")),
|
||||
self.assertEqual(t2.get_text(), "TOTO BEBE or not\n TOTO BEBE?\n")
|
||||
self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 0, 4, "xxx")),
|
||||
(t1.get_text(), value, Patch(0, 2, "To", "xxx")))
|
||||
self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 5, 9, "xxx")),
|
||||
self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 5, 9, "xxx")),
|
||||
(t1.get_text(), value, Patch(3, 5, "be", "xxx")))
|
||||
self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 18, 23, "xxx")),
|
||||
self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 18, 23, "xxx")),
|
||||
(t1.get_text(), value, Patch(14, 17, " to", "xxx")))
|
||||
# Match the entire second line
|
||||
self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 17, 29, "xxx")),
|
||||
self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 17, 29, "xxx")),
|
||||
(t1.get_text(), value, Patch(13, 21, " to be?", "xxx")))
|
||||
|
||||
def test_combiner(self):
|
||||
|
||||
@ -26,7 +26,7 @@ class TestUndo(test_engine.EngineTestCase):
|
||||
# Now undo just the first action. The list of undo DocActions for it does not mention the
|
||||
# newly added column, and fails to clean it up. This would leave the doc in an inconsistent
|
||||
# state, and we should not allow it.
|
||||
with self.assertRaisesRegexp(AssertionError,
|
||||
with self.assertRaisesRegex(AssertionError,
|
||||
re.compile(r"Internal schema inconsistent.*'NewCol'", re.S)):
|
||||
self.apply_undo_actions(out_actions1.undo)
|
||||
|
||||
@ -40,7 +40,7 @@ class TestUndo(test_engine.EngineTestCase):
|
||||
# In practice it's harmless: properly calculated fields get restored correct, and the private
|
||||
# metadata fields get brought up-to-date when used via Record interface, which is what we do
|
||||
# using this assertEqual().
|
||||
self.assertEqual([[r.id, r.tableId, map(int, r.columns)]
|
||||
self.assertEqual([[r.id, r.tableId, list(map(int, r.columns))]
|
||||
for r in self.engine.docmodel.tables.table.filter_records()], [
|
||||
[1, "Students", [1,2,4,5,6]],
|
||||
[2, "Schools", [10,12]],
|
||||
@ -73,7 +73,7 @@ class TestUndo(test_engine.EngineTestCase):
|
||||
|
||||
# The undo failed, and data should look as before the undo.
|
||||
self.engine.assert_schema_consistent()
|
||||
self.assertEqual([[r.id, r.tableId, map(int, r.columns)]
|
||||
self.assertEqual([[r.id, r.tableId, list(map(int, r.columns))]
|
||||
for r in self.engine.docmodel.tables.table.filter_records()], [
|
||||
[1, "Students", [1,2,4,5,6]],
|
||||
[2, "Schools", [10,12]],
|
||||
|
||||
@ -689,7 +689,7 @@ class TestUserActions(test_engine.EngineTestCase):
|
||||
|
||||
# Simple failure: bad action (last argument should be a dict). It shouldn't cause any actions
|
||||
# in the first place, just raise an exception about the argument being an int.
|
||||
with self.assertRaisesRegexp(AttributeError, r"'int'"):
|
||||
with self.assertRaisesRegex(AttributeError, r"'int'"):
|
||||
self.apply_user_action(['AddColumn', 'Address', "A", 17])
|
||||
|
||||
# Do some successful actions, just to make sure we know what they look like.
|
||||
@ -699,14 +699,14 @@ class TestUserActions(test_engine.EngineTestCase):
|
||||
)])
|
||||
|
||||
# More complicated: here some actions should succeed, but get reverted when a later one fails.
|
||||
with self.assertRaisesRegexp(AttributeError, r"'int'"):
|
||||
with self.assertRaisesRegex(AttributeError, r"'int'"):
|
||||
self.engine.apply_user_actions([useractions.from_repr(ua) for ua in (
|
||||
['UpdateRecord', 'Address', 11, {"city": "New York3"}],
|
||||
['AddColumn', 'Address', "C", {"isFormula": True}],
|
||||
['AddColumn', 'Address', "D", 17]
|
||||
)])
|
||||
|
||||
with self.assertRaisesRegexp(Exception, r"non-existent record #77"):
|
||||
with self.assertRaisesRegex(Exception, r"non-existent record #77"):
|
||||
self.engine.apply_user_actions([useractions.from_repr(ua) for ua in (
|
||||
['UpdateRecord', 'Address', 11, {"city": "New York4"}],
|
||||
['UpdateRecord', 'Address', 77, {"city": "Chicago"}],
|
||||
|
||||
@ -454,16 +454,16 @@
|
||||
// Check that Update Record on _grist_Tables_column properly triggers schema-change actions
|
||||
"USER_ACTIONS": [
|
||||
["UpdateRecord", "_grist_Tables_column", 3,
|
||||
{"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}],
|
||||
{"formula": "rec.firstName.upper() + ' ' + rec.lastName"}],
|
||||
["UpdateRecord", "_grist_Tables_column", 6,
|
||||
{"colId" : "shortSchool"}]
|
||||
],
|
||||
"ACTIONS": {
|
||||
"stored": [
|
||||
["ModifyColumn", "Students", "fullName",
|
||||
{"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}],
|
||||
{"formula": "rec.firstName.upper() + ' ' + rec.lastName"}],
|
||||
["UpdateRecord", "_grist_Tables_column", 3,
|
||||
{"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}],
|
||||
{"formula": "rec.firstName.upper() + ' ' + rec.lastName"}],
|
||||
["RenameColumn", "Students", "schoolShort", "shortSchool"],
|
||||
["UpdateRecord", "_grist_Tables_column", 6, {"colId": "shortSchool"}],
|
||||
["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8],
|
||||
@ -1921,14 +1921,14 @@
|
||||
// Check formula error handling
|
||||
"USER_ACTIONS" : [
|
||||
["ModifyColumn", "Students", "fullName", {"formula" : "!#@%&T#$UDSAIKVFsdhifzsk" }],
|
||||
["ModifyColumn", "Students", "schoolRegion", {"formula" : "5*len($firstName) / $fullNameLen" }]
|
||||
["ModifyColumn", "Students", "schoolRegion", {"formula" : "5*len($firstName) // $fullNameLen" }]
|
||||
],
|
||||
"ACTIONS" : {
|
||||
"stored" : [
|
||||
["ModifyColumn", "Students", "fullName", {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}],
|
||||
["UpdateRecord", "_grist_Tables_column", 3, {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}],
|
||||
["ModifyColumn", "Students", "schoolRegion", {"formula": "5*len($firstName) / $fullNameLen"}],
|
||||
["UpdateRecord", "_grist_Tables_column", 9, {"formula": "5*len($firstName) / $fullNameLen"}],
|
||||
["ModifyColumn", "Students", "schoolRegion", {"formula": "5*len($firstName) // $fullNameLen"}],
|
||||
["UpdateRecord", "_grist_Tables_column", 9, {"formula": "5*len($firstName) // $fullNameLen"}],
|
||||
["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8],
|
||||
{"fullName" : [["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"],
|
||||
["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"]]
|
||||
@ -2719,15 +2719,15 @@
|
||||
["APPLY", {
|
||||
"USER_ACTIONS": [
|
||||
// Basic tests
|
||||
["EvalCode", "print 'cats'", null],
|
||||
["EvalCode", "2 * 3 - 1 / 7 + 4", null],
|
||||
["EvalCode", "print('cats')", null],
|
||||
["EvalCode", "2 * 3 - 1 // 7 + 4", null],
|
||||
// Exception
|
||||
["EvalCode", "raise Exception('everything broke')", null],
|
||||
// Incomplete structure
|
||||
["EvalCode", "for x in range(1, 100):", null],
|
||||
// Re-evaluation
|
||||
["EvalCode", "print 'cats'", 1],
|
||||
["EvalCode", "print 'dogs'", 1],
|
||||
["EvalCode", "print('cats')", 1],
|
||||
["EvalCode", "print('dogs')", 1],
|
||||
// Function definition
|
||||
["EvalCode", "def foo(x):\n\treturn x * 10\n", null],
|
||||
["EvalCode", "foo(10)", null]
|
||||
@ -2735,15 +2735,15 @@
|
||||
"ACTIONS" : {
|
||||
"stored" : [
|
||||
["AddRecord", "_grist_REPL_Hist", 1,
|
||||
{"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}],
|
||||
{"code": "print('cats')", "errorText": "", "outputText": "cats\n"}],
|
||||
["AddRecord", "_grist_REPL_Hist", 2,
|
||||
{"code": "2 * 3 - 1 / 7 + 4", "errorText": "", "outputText": "10\n"}],
|
||||
{"code": "2 * 3 - 1 // 7 + 4", "errorText": "", "outputText": "10\n"}],
|
||||
["AddRecord", "_grist_REPL_Hist", 3,
|
||||
{"code": "raise Exception('everything broke')", "errorText": "Traceback (most recent call last):\n File \"<input>\", line 1, in <module>\nException: everything broke\n", "outputText": ""}],
|
||||
["UpdateRecord", "_grist_REPL_Hist", 1,
|
||||
{"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}],
|
||||
{"code": "print('cats')", "errorText": "", "outputText": "cats\n"}],
|
||||
["UpdateRecord", "_grist_REPL_Hist", 1,
|
||||
{"code": "print 'dogs'", "errorText": "", "outputText": "dogs\n"}],
|
||||
{"code": "print('dogs')", "errorText": "", "outputText": "dogs\n"}],
|
||||
["AddRecord", "_grist_REPL_Hist", 4,
|
||||
{"code": "def foo(x):\n\treturn x * 10\n", "errorText": "", "outputText": ""}],
|
||||
["AddRecord", "_grist_REPL_Hist", 5,
|
||||
@ -2755,9 +2755,9 @@
|
||||
["RemoveRecord", "_grist_REPL_Hist", 2],
|
||||
["RemoveRecord", "_grist_REPL_Hist", 3],
|
||||
["UpdateRecord", "_grist_REPL_Hist", 1,
|
||||
{"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}],
|
||||
{"code": "print('cats')", "errorText": "", "outputText": "cats\n"}],
|
||||
["UpdateRecord", "_grist_REPL_Hist", 1,
|
||||
{"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}],
|
||||
{"code": "print('cats')", "errorText": "", "outputText": "cats\n"}],
|
||||
["RemoveRecord", "_grist_REPL_Hist", 4],
|
||||
["RemoveRecord", "_grist_REPL_Hist", 5]
|
||||
],
|
||||
|
||||
@ -3,6 +3,8 @@ import math
|
||||
import os
|
||||
import re
|
||||
|
||||
import six
|
||||
|
||||
import actions
|
||||
import logger
|
||||
|
||||
@ -103,7 +105,7 @@ def parse_test_sample(obj, samples={}):
|
||||
}
|
||||
|
||||
data = {t: table_data_from_rows(t, data[0], data[1:])
|
||||
for t, data in obj["DATA"].iteritems()}
|
||||
for t, data in six.iteritems(obj["DATA"])}
|
||||
return {"SCHEMA": schema, "DATA": data}
|
||||
|
||||
|
||||
|
||||
@ -11,6 +11,8 @@ import bisect
|
||||
import re
|
||||
from collections import namedtuple
|
||||
|
||||
import six
|
||||
|
||||
Patch = namedtuple('Patch', ('start', 'end', 'old_text', 'new_text'))
|
||||
|
||||
line_start_re = re.compile(r'^', re.M)
|
||||
@ -176,4 +178,4 @@ class Combiner(Builder):
|
||||
offset = self._offsets[start_index - 1]
|
||||
part = self._parts[start_index - 1]
|
||||
in_patch = Patch(patch.start - offset, patch.end - offset, patch.old_text, patch.new_text)
|
||||
return None if isinstance(part, basestring) else part.map_back_patch(in_patch)
|
||||
return None if isinstance(part, six.string_types) else part.map_back_patch(in_patch)
|
||||
|
||||
@ -14,6 +14,8 @@ value previously set, since the "right" dataset is "single" values), m.lookup_le
|
||||
that value, and m.lookup_right(value) returns a `set` of keys that map to the value.
|
||||
"""
|
||||
|
||||
import six
|
||||
|
||||
# Special sentinel value which can never be legitimately stored in TwoWayMap, to easily tell the
|
||||
# difference between a present and absent value.
|
||||
_NIL = object()
|
||||
@ -47,6 +49,8 @@ class TwoWayMap(object):
|
||||
def __nonzero__(self):
|
||||
return bool(self._fwd)
|
||||
|
||||
__bool__ = __nonzero__
|
||||
|
||||
def lookup_left(self, left, default=None):
|
||||
""" Returns the value(s) on the right corresponding to the given value on the left. """
|
||||
return self._fwd.get(left, default)
|
||||
@ -65,11 +69,11 @@ class TwoWayMap(object):
|
||||
|
||||
def left_all(self):
|
||||
""" Returns an iterable over all values on the left."""
|
||||
return self._fwd.iterkeys()
|
||||
return six.iterkeys(self._fwd)
|
||||
|
||||
def right_all(self):
|
||||
""" Returns an iterable over all values on the right."""
|
||||
return self._bwd.iterkeys()
|
||||
return six.iterkeys(self._bwd)
|
||||
|
||||
def insert(self, left, right):
|
||||
""" Insert the (left, right) value pair. """
|
||||
|
||||
@ -4,6 +4,8 @@ import re
|
||||
import json
|
||||
import sys
|
||||
|
||||
import six
|
||||
|
||||
import acl
|
||||
from acl_formula import parse_acl_formula_json
|
||||
import actions
|
||||
@ -45,7 +47,7 @@ _modify_col_schema_props = {'type', 'formula', 'isFormula'}
|
||||
# A few generic helpers.
|
||||
def select_keys(dict_obj, keys):
|
||||
"""Return copy of dict_obj containing only the given keys."""
|
||||
return {k: v for k, v in dict_obj.iteritems() if k in keys}
|
||||
return {k: v for k, v in six.iteritems(dict_obj) if k in keys}
|
||||
|
||||
def has_value(dict_obj, key, value):
|
||||
"""Returns True if dict_obj contains key, and its value is value."""
|
||||
@ -78,7 +80,7 @@ def useraction(method):
|
||||
"""
|
||||
Decorator for a method, which creates an action class with the same name and arguments.
|
||||
"""
|
||||
code = method.func_code
|
||||
code = method.__code__
|
||||
name = method.__name__
|
||||
cls = namedtuple(name, code.co_varnames[1:code.co_argcount])
|
||||
setattr(_current_module, name, cls)
|
||||
@ -148,7 +150,7 @@ class UserActions(object):
|
||||
# Map of methods implementing particular (action_name, table_id) combinations. It mirrors
|
||||
# global _action_method_overrides, but with methods *bound* to this UserActions instance.
|
||||
self._overrides = {key: method.__get__(self, UserActions)
|
||||
for key, method in _action_method_overrides.iteritems()}
|
||||
for key, method in six.iteritems(_action_method_overrides)}
|
||||
|
||||
def _do_doc_action(self, action):
|
||||
if hasattr(action, 'simplify'):
|
||||
@ -169,7 +171,7 @@ class UserActions(object):
|
||||
for i, row_id in enumerate(row_ids):
|
||||
rec = table.get_record(row_id)
|
||||
yield ((i, rec) if col_values is None else
|
||||
(i, rec, {k: v[i] for k, v in col_values.iteritems()}))
|
||||
(i, rec, {k: v[i] for k, v in six.iteritems(col_values)}))
|
||||
|
||||
def _collect_back_references(self, table_recs):
|
||||
"""
|
||||
@ -273,13 +275,13 @@ class UserActions(object):
|
||||
@useraction
|
||||
def AddRecord(self, table_id, row_id, column_values):
|
||||
return self.BulkAddRecord(
|
||||
table_id, [row_id], {key: [val] for key, val in column_values.iteritems()}
|
||||
table_id, [row_id], {key: [val] for key, val in six.iteritems(column_values)}
|
||||
)[0]
|
||||
|
||||
@useraction
|
||||
def BulkAddRecord(self, table_id, row_ids, column_values):
|
||||
column_values = actions.decode_bulk_values(column_values)
|
||||
for col_id, values in column_values.iteritems():
|
||||
for col_id, values in six.iteritems(column_values):
|
||||
self._ensure_column_accepts_data(table_id, col_id, values)
|
||||
method = self._overrides.get(('BulkAddRecord', table_id), self.doBulkAddOrReplace)
|
||||
return method(table_id, row_ids, column_values)
|
||||
@ -339,7 +341,7 @@ class UserActions(object):
|
||||
def _addACLRules(self, table_id, row_ids, col_values):
|
||||
# Automatically populate aclFormulaParsed value by parsing aclFormula.
|
||||
if 'aclFormula' in col_values:
|
||||
col_values['aclFormulaParsed'] = map(parse_acl_formula_json, col_values['aclFormula'])
|
||||
col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']]
|
||||
return self.doBulkAddOrReplace(table_id, row_ids, col_values)
|
||||
|
||||
#----------------------------------------
|
||||
@ -371,7 +373,7 @@ class UserActions(object):
|
||||
@useraction
|
||||
def UpdateRecord(self, table_id, row_id, columns):
|
||||
self.BulkUpdateRecord(table_id, [row_id],
|
||||
{key: [col] for key, col in columns.iteritems()})
|
||||
{key: [col] for key, col in six.iteritems(columns)})
|
||||
|
||||
@useraction
|
||||
def BulkUpdateRecord(self, table_id, row_ids, columns):
|
||||
@ -380,7 +382,7 @@ class UserActions(object):
|
||||
# Handle special tables, updates to which imply metadata actions.
|
||||
|
||||
# Check that the update is valid.
|
||||
for col_id, values in columns.iteritems():
|
||||
for col_id, values in six.iteritems(columns):
|
||||
self._ensure_column_accepts_data(table_id, col_id, values)
|
||||
|
||||
# Additionally check that we are not trying to modify group-by values in a summary column
|
||||
@ -413,7 +415,7 @@ class UserActions(object):
|
||||
|
||||
@override_action('BulkUpdateRecord', '_grist_Tables')
|
||||
def _updateTableRecords(self, table_id, row_ids, col_values):
|
||||
avoid_tableid_set = set(self._engine.tables.viewkeys())
|
||||
avoid_tableid_set = set(self._engine.tables)
|
||||
update_pairs = []
|
||||
for i, rec, values in self._bulk_action_iter(table_id, row_ids, col_values):
|
||||
update_pairs.append((rec, values))
|
||||
@ -451,9 +453,9 @@ class UserActions(object):
|
||||
if table_renames:
|
||||
# Build up a dictionary mapping col_ref of each affected formula to the new formula text.
|
||||
formula_updates = self._prepare_formula_renames(
|
||||
{(old, None): new for (old, new) in table_renames.iteritems()})
|
||||
{(old, None): new for (old, new) in six.iteritems(table_renames)})
|
||||
# Add the changes to the dict of col_updates. sort for reproducible order.
|
||||
for col_rec, new_formula in sorted(formula_updates.iteritems()):
|
||||
for col_rec, new_formula in sorted(six.iteritems(formula_updates)):
|
||||
col_updates.setdefault(col_rec, {})['formula'] = new_formula
|
||||
|
||||
# If a table changes to onDemand, any empty columns (formula columns with no set formula)
|
||||
@ -464,7 +466,7 @@ class UserActions(object):
|
||||
for col in empty_cols:
|
||||
col_updates.setdefault(col, {}).update(isFormula=False, type='Text')
|
||||
|
||||
for col, values in col_updates.iteritems():
|
||||
for col, values in six.iteritems(col_updates):
|
||||
if 'type' in values:
|
||||
self.doModifyColumn(col.tableId, col.colId, {'type': 'Int'})
|
||||
|
||||
@ -482,7 +484,7 @@ class UserActions(object):
|
||||
# Internal functions are used to prevent unintended additional changes from occurring.
|
||||
# Specifically, this prevents widgetOptions and displayCol from being cleared as a side
|
||||
# effect of the column type change.
|
||||
for col, values in col_updates.iteritems():
|
||||
for col, values in six.iteritems(col_updates):
|
||||
self.doModifyColumn(col.tableId, col.colId, values)
|
||||
self.doBulkUpdateFromPairs('_grist_Tables_column', col_updates.items())
|
||||
make_acl_updates()
|
||||
@ -516,7 +518,7 @@ class UserActions(object):
|
||||
|
||||
# Collect all renamings that we are about to apply.
|
||||
renames = {(c.parentId.tableId, c.colId): values['colId']
|
||||
for c, values in col_updates.iteritems()
|
||||
for c, values in six.iteritems(col_updates)
|
||||
if has_diff_value(values, 'colId', c.colId)}
|
||||
|
||||
if renames:
|
||||
@ -524,7 +526,7 @@ class UserActions(object):
|
||||
formula_updates = self._prepare_formula_renames(renames)
|
||||
|
||||
# For any affected columns, include the formula into the update.
|
||||
for col_rec, new_formula in sorted(formula_updates.iteritems()):
|
||||
for col_rec, new_formula in sorted(six.iteritems(formula_updates)):
|
||||
col_updates.setdefault(col_rec, {}).setdefault('formula', new_formula)
|
||||
|
||||
update_pairs = col_updates.items()
|
||||
@ -534,7 +536,7 @@ class UserActions(object):
|
||||
if col.summarySourceCol:
|
||||
underlying = col_updates.get(col.summarySourceCol, {})
|
||||
if not all(value == getattr(col, key) or has_value(underlying, key, value)
|
||||
for key, value in values.iteritems()):
|
||||
for key, value in six.iteritems(values)):
|
||||
raise ValueError("Cannot modify summary group-by column '%s'" % col.colId)
|
||||
|
||||
make_acl_updates = acl.prepare_acl_col_renames(self._docmodel, self, renames)
|
||||
@ -576,7 +578,7 @@ class UserActions(object):
|
||||
def _updateACLRules(self, table_id, row_ids, col_values):
|
||||
# Automatically populate aclFormulaParsed value by parsing aclFormula.
|
||||
if 'aclFormula' in col_values:
|
||||
col_values['aclFormulaParsed'] = map(parse_acl_formula_json, col_values['aclFormula'])
|
||||
col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']]
|
||||
return self.doBulkUpdateRecord(table_id, row_ids, col_values)
|
||||
|
||||
def _prepare_formula_renames(self, renames):
|
||||
@ -605,7 +607,7 @@ class UserActions(object):
|
||||
# Apply the collected patches to each affected formula, converting to unicode to apply the
|
||||
# patches and back to byte string for how we maintain string values.
|
||||
result = {}
|
||||
for col_rec, patches in patches_map.iteritems():
|
||||
for col_rec, patches in six.iteritems(patches_map):
|
||||
formula = col_rec.formula.decode('utf8')
|
||||
replacer = textbuilder.Replacer(textbuilder.Text(formula), patches)
|
||||
result[col_rec] = replacer.get_text().encode('utf8')
|
||||
@ -1066,7 +1068,7 @@ class UserActions(object):
|
||||
# metadata record. We implement the former interface by forwarding to the latter.
|
||||
col = self._docmodel.get_column_rec(table_id, col_id)
|
||||
|
||||
update_values = {k: v for k, v in col_info.iteritems() if k in _modifiable_col_fields}
|
||||
update_values = {k: v for k, v in six.iteritems(col_info) if k in _modifiable_col_fields}
|
||||
if '_position' in col_info:
|
||||
update_values['parentPos'] = col_info['_position']
|
||||
self._docmodel.update([col], **update_values)
|
||||
@ -1093,7 +1095,7 @@ class UserActions(object):
|
||||
old_col_info = schema.col_to_dict(self._engine.schema[table_id].columns[col_id],
|
||||
include_id=False)
|
||||
|
||||
col_info = {k: v for k, v in col_info.iteritems() if old_col_info.get(k, v) != v}
|
||||
col_info = {k: v for k, v in six.iteritems(col_info) if old_col_info.get(k, v) != v}
|
||||
if not col_info:
|
||||
log.info("useractions.ModifyColumn is a noop")
|
||||
return
|
||||
@ -1182,7 +1184,7 @@ class UserActions(object):
|
||||
|
||||
# Get the values from the columns and check which have changed.
|
||||
all_row_ids = list(table.row_ids)
|
||||
all_src_values = map(src_column.raw_get, all_row_ids)
|
||||
all_src_values = [src_column.raw_get(r) for r in all_row_ids]
|
||||
|
||||
dst_column = table.get_column(dst_col_id)
|
||||
changed_rows, changed_values = [], []
|
||||
@ -1230,7 +1232,7 @@ class UserActions(object):
|
||||
Add the given table with columns without creating views.
|
||||
"""
|
||||
# If needed, transform table_id into a valid identifier, and add a suffix to make it unique.
|
||||
table_id = identifiers.pick_table_ident(table_id, avoid=self._engine.tables.viewkeys())
|
||||
table_id = identifiers.pick_table_ident(table_id, avoid=six.viewkeys(self._engine.tables))
|
||||
|
||||
# Sanitize and de-duplicate column identifiers.
|
||||
col_ids = [c['id'] for c in columns]
|
||||
|
||||
@ -161,7 +161,7 @@ class Text(BaseColumnType):
|
||||
|
||||
@classmethod
|
||||
def is_right_type(cls, value):
|
||||
return isinstance(value, (basestring, NoneType))
|
||||
return isinstance(value, (six.string_types, NoneType))
|
||||
|
||||
@classmethod
|
||||
def typeConvert(cls, value):
|
||||
@ -203,11 +203,11 @@ class Bool(BaseColumnType):
|
||||
# recognize. Everything else will result in alttext.
|
||||
if not value:
|
||||
return False
|
||||
if isinstance(value, (float, int, long)):
|
||||
if isinstance(value, (float, six.integer_types)):
|
||||
return True
|
||||
if isinstance(value, AltText):
|
||||
value = str(value)
|
||||
if isinstance(value, basestring):
|
||||
if isinstance(value, six.string_types):
|
||||
if value.lower() in ("false", "no", "0"):
|
||||
return False
|
||||
if value.lower() in ("true", "yes", "1"):
|
||||
@ -235,7 +235,7 @@ class Int(BaseColumnType):
|
||||
|
||||
@classmethod
|
||||
def is_right_type(cls, value):
|
||||
return value is None or (isinstance(value, (int, long)) and not isinstance(value, bool) and
|
||||
return value is None or (isinstance(value, six.integer_types) and not isinstance(value, bool) and
|
||||
objtypes.is_int_short(value))
|
||||
|
||||
|
||||
@ -252,7 +252,7 @@ class Numeric(BaseColumnType):
|
||||
# TODO: Python distinguishes ints from floats, while JS only has floats. A value that can be
|
||||
# interpreted as an int will upon being entered have type 'float', but after database reload
|
||||
# will have type 'int'.
|
||||
return isinstance(value, (float, int, long, NoneType)) and not isinstance(value, bool)
|
||||
return isinstance(value, (float, six.integer_types, NoneType)) and not isinstance(value, bool)
|
||||
|
||||
|
||||
class Date(Numeric):
|
||||
@ -267,9 +267,9 @@ class Date(Numeric):
|
||||
return moment.dt_to_ts(value)
|
||||
elif isinstance(value, datetime.date):
|
||||
return moment.date_to_ts(value)
|
||||
elif isinstance(value, (float, int, long)):
|
||||
elif isinstance(value, (float, six.integer_types)):
|
||||
return float(value)
|
||||
elif isinstance(value, basestring):
|
||||
elif isinstance(value, six.string_types):
|
||||
# We also accept a date in ISO format (YYYY-MM-DD), the time portion is optional and ignored
|
||||
return moment.parse_iso_date(value)
|
||||
else:
|
||||
@ -277,7 +277,7 @@ class Date(Numeric):
|
||||
|
||||
@classmethod
|
||||
def is_right_type(cls, value):
|
||||
return isinstance(value, (float, int, long, NoneType))
|
||||
return isinstance(value, (float, six.integer_types, NoneType))
|
||||
|
||||
@classmethod
|
||||
def typeConvert(cls, value, date_format, timezone='UTC'): # pylint: disable=arguments-differ
|
||||
@ -306,9 +306,9 @@ class DateTime(Date):
|
||||
return moment.dt_to_ts(value, self.timezone)
|
||||
elif isinstance(value, datetime.date):
|
||||
return moment.date_to_ts(value, self.timezone)
|
||||
elif isinstance(value, (float, int, long)):
|
||||
elif isinstance(value, (float, six.integer_types)):
|
||||
return float(value)
|
||||
elif isinstance(value, basestring):
|
||||
elif isinstance(value, six.string_types):
|
||||
# We also accept a datetime in ISO format (YYYY-MM-DD[T]HH:mm:ss)
|
||||
return moment.parse_iso(value, self.timezone)
|
||||
else:
|
||||
@ -330,7 +330,7 @@ class ChoiceList(BaseColumnType):
|
||||
def do_convert(self, value):
|
||||
if not value:
|
||||
return None
|
||||
elif isinstance(value, basestring):
|
||||
elif isinstance(value, six.string_types):
|
||||
# If it's a string that looks like JSON, try to parse it as such.
|
||||
if value.startswith('['):
|
||||
try:
|
||||
@ -345,11 +345,11 @@ class ChoiceList(BaseColumnType):
|
||||
@classmethod
|
||||
def is_right_type(cls, value):
|
||||
return value is None or (isinstance(value, (tuple, list)) and
|
||||
all(isinstance(item, basestring) for item in value))
|
||||
all(isinstance(item, six.string_types) for item in value))
|
||||
|
||||
@classmethod
|
||||
def typeConvert(cls, value):
|
||||
if isinstance(value, basestring) and not value.startswith('['):
|
||||
if isinstance(value, six.string_types) and not value.startswith('['):
|
||||
# Try to parse as CSV. If this doesn't work, we'll still try usual conversions later.
|
||||
try:
|
||||
tags = next(csv.reader([value]))
|
||||
@ -383,7 +383,7 @@ class PositionNumber(BaseColumnType):
|
||||
@classmethod
|
||||
def is_right_type(cls, value):
|
||||
# Same as Numeric, but does not support None.
|
||||
return isinstance(value, (float, int, long)) and not isinstance(value, bool)
|
||||
return isinstance(value, (float, six.integer_types)) and not isinstance(value, bool)
|
||||
|
||||
|
||||
class ManualSortPos(PositionNumber):
|
||||
@ -411,7 +411,7 @@ class Id(BaseColumnType):
|
||||
|
||||
@classmethod
|
||||
def is_right_type(cls, value):
|
||||
return (isinstance(value, (int, long)) and not isinstance(value, bool) and
|
||||
return (isinstance(value, six.integer_types) and not isinstance(value, bool) and
|
||||
objtypes.is_int_short(value))
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user