From 16f297a250b7fb6d6b3029608f053a365aa60f41 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Tue, 22 Jun 2021 17:12:25 +0200 Subject: [PATCH] (core) Simple Python 3 compatibility changes Summary: Changes that move towards python 3 compatibility that are easy to review without much thought Test Plan: The tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2873 --- sandbox/gen_js_schema.py | 22 +-- sandbox/grist/acl.py | 4 +- sandbox/grist/acl_formula.py | 5 + sandbox/grist/action_obj.py | 12 +- sandbox/grist/action_summary.py | 4 +- sandbox/grist/actions.py | 19 ++- sandbox/grist/autocomplete_context.py | 14 +- sandbox/grist/codebuilder.py | 8 +- sandbox/grist/column.py | 4 +- sandbox/grist/csv_patch.py | 1 + sandbox/grist/docactions.py | 20 +-- sandbox/grist/docmodel.py | 12 +- sandbox/grist/engine.py | 61 +++---- sandbox/grist/functions/__init__.py | 16 +- sandbox/grist/functions/date.py | 4 +- sandbox/grist/functions/info.py | 4 +- sandbox/grist/functions/logical.py | 4 +- sandbox/grist/functions/lookup.py | 17 +- sandbox/grist/functions/math.py | 14 +- sandbox/grist/functions/schedule.py | 2 +- sandbox/grist/functions/stats.py | 14 +- sandbox/grist/functions/test_schedule.py | 46 +++--- sandbox/grist/functions/text.py | 8 +- sandbox/grist/gencode.py | 14 +- sandbox/grist/gpath.py | 3 + sandbox/grist/import_actions.py | 3 +- sandbox/grist/imports/test_messytables.py | 2 +- sandbox/grist/lookup.py | 4 +- sandbox/grist/main.py | 8 +- sandbox/grist/migrations.py | 19 ++- sandbox/grist/moment.py | 7 +- sandbox/grist/objtypes.py | 7 +- sandbox/grist/records.py | 4 + sandbox/grist/relabeling.py | 7 +- sandbox/grist/repl.py | 2 +- sandbox/grist/runtests.py | 6 +- sandbox/grist/sandbox.py | 4 +- sandbox/grist/schema.py | 5 +- sandbox/grist/summary.py | 10 +- sandbox/grist/table.py | 19 ++- sandbox/grist/table_data_set.py | 18 ++- sandbox/grist/test_acl_formula.py | 15 +- sandbox/grist/test_codebuilder.py | 8 +- sandbox/grist/test_column_actions.py | 2 +- sandbox/grist/test_docmodel.py | 4 +- sandbox/grist/test_engine.py | 22 +-- sandbox/grist/test_formula_error.py | 16 +- sandbox/grist/test_gencode.py | 2 + sandbox/grist/test_gpath.py | 34 ++-- sandbox/grist/test_import_actions.py | 14 +- sandbox/grist/test_match_counter.py | 6 +- sandbox/grist/test_migrations.py | 8 +- sandbox/grist/test_moment.py | 6 +- sandbox/grist/test_record_func.py | 185 +++++++++++----------- sandbox/grist/test_relabeling.py | 13 +- sandbox/grist/test_summary2.py | 24 +-- sandbox/grist/test_table_data_set.py | 22 +-- sandbox/grist/test_textbuilder.py | 12 +- sandbox/grist/test_undo.py | 6 +- sandbox/grist/test_useractions.py | 6 +- sandbox/grist/testscript.json | 32 ++-- sandbox/grist/testutil.py | 4 +- sandbox/grist/textbuilder.py | 4 +- sandbox/grist/twowaymap.py | 8 +- sandbox/grist/useractions.py | 48 +++--- sandbox/grist/usertypes.py | 30 ++-- 66 files changed, 551 insertions(+), 437 deletions(-) diff --git a/sandbox/gen_js_schema.py b/sandbox/gen_js_schema.py index 4154ec48..5fd9cc00 100644 --- a/sandbox/gen_js_schema.py +++ b/sandbox/gen_js_schema.py @@ -24,29 +24,29 @@ def get_ts_type(col_type): return _ts_types.get(col_type, "CellValue") def main(): - print """ + print(""" /*** THIS FILE IS AUTO-GENERATED BY %s ***/ // tslint:disable:object-literal-key-quotes export const schema = { -""" % __file__ +""" % __file__) for table in schema.schema_create_actions(): - print ' "%s": {' % table.table_id + print(' "%s": {' % table.table_id) for column in table.columns: - print ' %-20s: "%s",' % (column['id'], column['type']) - print ' },\n' + print(' %-20s: "%s",' % (column['id'], column['type'])) + print(' },\n') - print """}; + print("""}; export interface SchemaTypes { -""" +""") for table in schema.schema_create_actions(): - print ' "%s": {' % table.table_id + print(' "%s": {' % table.table_id) for column in table.columns: - print ' %s: %s;' % (column['id'], get_ts_type(column['type'])) - print ' };\n' - print "}" + print(' %s: %s;' % (column['id'], get_ts_type(column['type']))) + print(' };\n') + print("}") if __name__ == '__main__': main() diff --git a/sandbox/grist/acl.py b/sandbox/grist/acl.py index bc3bd1cc..f16064da 100644 --- a/sandbox/grist/acl.py +++ b/sandbox/grist/acl.py @@ -67,7 +67,7 @@ def prepare_acl_table_renames(docmodel, useractions, table_renames_dict): if rule_info.get("tableId") in table_renames_dict: rule_info["tableId"] = table_renames_dict[rule_info.get("tableId")] rule_updates.append((rule_rec, {'userAttributes': json.dumps(rule_info)})) - except Exception, e: + except Exception as e: log.warn("Error examining aclRule: %s" % (e,)) def do_renames(): @@ -103,7 +103,7 @@ def prepare_acl_col_renames(docmodel, useractions, col_renames_dict): if new_col_id: rule_info["lookupColId"] = new_col_id rule_updates.append((rule_rec, {'userAttributes': json.dumps(rule_info)})) - except Exception, e: + except Exception as e: log.warn("Error examining aclRule: %s" % (e,)) # Go through again checking if anything in ACL formulas is affected by the rename. diff --git a/sandbox/grist/acl_formula.py b/sandbox/grist/acl_formula.py index 29ed80c2..2d3ad887 100644 --- a/sandbox/grist/acl_formula.py +++ b/sandbox/grist/acl_formula.py @@ -104,6 +104,11 @@ class _TreeConverter(ast.NodeVisitor): return ["Const", named_constants[node.id]] return ["Name", node.id] + def visit_Constant(self, node): + return ["Const", node.value] + + visit_NameConstant = visit_Constant + def visit_Attribute(self, node): return ["Attr", self.visit(node.value), node.attr] diff --git a/sandbox/grist/action_obj.py b/sandbox/grist/action_obj.py index 61be77ae..654329af 100644 --- a/sandbox/grist/action_obj.py +++ b/sandbox/grist/action_obj.py @@ -54,9 +54,9 @@ class ActionGroup(object): def get_repr(self): return { - "calc": map(actions.get_action_repr, self.calc), - "stored": map(actions.get_action_repr, self.stored), - "undo": map(actions.get_action_repr, self.undo), + "calc": [actions.get_action_repr(a) for a in self.calc], + "stored": [actions.get_action_repr(a) for a in self.stored], + "undo": [actions.get_action_repr(a) for a in self.undo], "direct": self.direct, "retValues": self.retValues } @@ -64,9 +64,9 @@ class ActionGroup(object): @classmethod def from_json_obj(cls, data): ag = ActionGroup() - ag.calc = map(actions.action_from_repr, data.get('calc', [])) - ag.stored = map(actions.action_from_repr, data.get('stored', [])) - ag.undo = map(actions.action_from_repr, data.get('undo', [])) + ag.calc = [actions.action_from_repr(a) for a in data.get('calc', [])] + ag.stored = [actions.action_from_repr(a) for a in data.get('stored', [])] + ag.undo = [actions.action_from_repr(a) for a in data.get('undo', [])] ag.retValues = data.get('retValues', []) return ag diff --git a/sandbox/grist/action_summary.py b/sandbox/grist/action_summary.py index d2218c53..9bc3e07b 100644 --- a/sandbox/grist/action_summary.py +++ b/sandbox/grist/action_summary.py @@ -4,6 +4,8 @@ It's used for collecting calculated values for formula columns. """ from collections import namedtuple +import six + import actions from objtypes import equal_encoding @@ -76,7 +78,7 @@ class ActionSummary(object): """ if not column_delta: return - full_row_ids = sorted(r for r, (before, after) in column_delta.iteritems() + full_row_ids = sorted(r for r, (before, after) in six.iteritems(column_delta) if not equal_encoding(before, after)) defunct = is_defunct(table_id) or is_defunct(col_id) diff --git a/sandbox/grist/actions.py b/sandbox/grist/actions.py index 14cfccb1..ab932862 100644 --- a/sandbox/grist/actions.py +++ b/sandbox/grist/actions.py @@ -8,6 +8,8 @@ When communicating with Node, docActions are represented as arrays [actionName, from collections import namedtuple import inspect +import six + import objtypes def _eq_with_type(self, other): @@ -86,7 +88,7 @@ def _add_simplify(SingleActionType, BulkActionType): else: def get_first(self): return SingleActionType(self.table_id, self.row_ids[0], - { key: col[0] for key, col in self.columns.iteritems()}) + { key: col[0] for key, col in six.iteritems(self.columns)}) def simplify(self): return None if not self.row_ids else (get_first(self) if len(self.row_ids) == 1 else self) @@ -128,7 +130,7 @@ def convert_recursive_helper(converter, data): return convert_recursive_helper(my_convert, data) """ if isinstance(data, dict): - return {converter(k): converter(v) for k, v in data.iteritems()} + return {converter(k): converter(v) for k, v in six.iteritems(data)} elif isinstance(data, list): return [converter(el) for el in data] elif isinstance(data, tuple): @@ -142,10 +144,12 @@ def convert_action_values(converter, action): """ if isinstance(action, (AddRecord, UpdateRecord)): return type(action)(action.table_id, action.row_id, - {k: converter(v) for k, v in action.columns.iteritems()}) + {k: converter(v) for k, v in six.iteritems(action.columns)}) if isinstance(action, (BulkAddRecord, BulkUpdateRecord, ReplaceTableData, TableData)): - return type(action)(action.table_id, action.row_ids, - {k: map(converter, v) for k, v in action.columns.iteritems()}) + return type(action)( + action.table_id, action.row_ids, + {k: [converter(value) for value in values] for k, values in six.iteritems(action.columns)} + ) return action def convert_recursive_in_action(converter, data): @@ -173,7 +177,10 @@ def decode_bulk_values(bulk_values, decoder=objtypes.decode_object): Decode objects in values of the form {col_id: array_of_values}, as present in bulk DocActions and UserActions. """ - return {k: map(decoder, v) for (k, v) in bulk_values.iteritems()} + return { + k: [decoder(value) for value in values] + for k, values in six.iteritems(bulk_values) + } def transpose_bulk_action(bulk_action): """ diff --git a/sandbox/grist/autocomplete_context.py b/sandbox/grist/autocomplete_context.py index 19fdd0ca..fce1c102 100644 --- a/sandbox/grist/autocomplete_context.py +++ b/sandbox/grist/autocomplete_context.py @@ -4,10 +4,12 @@ Helper class for handling formula autocomplete. It's intended to use with rlcompleter.Completer. It allows finding global names using lowercase searches, and adds function usage information to some results. """ -import __builtin__ +from six.moves import builtins import inspect from collections import namedtuple +import six + # funcname is the function name, e.g. "MAX" # argspec is the signature, e.g. "(arg, *more_args)" # isgrist is a boolean for whether this function should be in Grist documentation. @@ -16,7 +18,7 @@ Completion = namedtuple('Completion', ['funcname', 'argspec', 'isgrist']) def is_grist_func(func): try: return inspect.getmodule(func).__name__.startswith('functions.') - except Exception, e: + except Exception as e: return e class AutocompleteContext(object): @@ -24,7 +26,7 @@ class AutocompleteContext(object): # rlcompleter is case-sensitive. This is hard to work around while maintaining attribute # lookups. As a middle ground, we only introduce lowercase versions of all global names. self._context = { - key: value for key, value in usercode_context.iteritems() + key: value for key, value in six.iteritems(usercode_context) # Don't propose unimplemented functions in autocomplete if not (value and callable(value) and getattr(value, 'unimplemented', None)) } @@ -32,7 +34,7 @@ class AutocompleteContext(object): # Prepare detailed Completion objects for functions where we can supply more info. # TODO It would be nice to include builtin functions too, but getargspec doesn't work there. self._functions = {} - for key, value in self._context.iteritems(): + for key, value in six.iteritems(self._context): if value and callable(value): argspec = inspect.formatargspec(*inspect.getargspec(value)) self._functions[key] = Completion(key, argspec, is_grist_func(value)) @@ -47,7 +49,7 @@ class AutocompleteContext(object): lower = key.lower() if lower == key: continue - if lower not in self._context and lower not in __builtin__.__dict__: + if lower not in self._context and lower not in builtins.__dict__: self._lowercase[lower] = key else: # This is still good enough to find a match for, and translate back to the original. @@ -59,7 +61,7 @@ class AutocompleteContext(object): self._lowercase[lower] = key # Add the lowercase names to the context, and to the detailed completions in _functions. - for lower, key in self._lowercase.iteritems(): + for lower, key in six.iteritems(self._lowercase): self._context[lower] = self._context[key] if key in self._functions: self._functions[lower] = self._functions[key] diff --git a/sandbox/grist/codebuilder.py b/sandbox/grist/codebuilder.py index ec73ade6..e969db84 100644 --- a/sandbox/grist/codebuilder.py +++ b/sandbox/grist/codebuilder.py @@ -127,7 +127,7 @@ def _create_syntax_error_code(builder, input_text, err): def infer(node): try: return next(node.infer(), None) - except astroid.exceptions.InferenceError, e: + except astroid.exceptions.InferenceError as e: return "InferenceError on %r: %r" % (node, e) @@ -137,7 +137,7 @@ def _is_table(node): """ Return true if obj is a class defining a user table. """ - return (isinstance(node, astroid.nodes.Class) and node.decorators and + return (isinstance(node, astroid.nodes.ClassDef) and node.decorators and node.decorators.nodes[0].as_string() == 'grist.UserTable') @@ -202,7 +202,7 @@ class InferReferenceFormula(InferenceTip): Inference helper to treat functions decorated with `grist.formulaType(grist.Reference("Foo"))` as returning instances of table `Foo`. """ - node_class = astroid.nodes.Function + node_class = astroid.nodes.FunctionDef @classmethod def filter(cls, node): @@ -320,4 +320,4 @@ def parse_grist_names(builder): start, end = tok.startpos, tok.endpos parsed_names.append(make_tuple(start, end, obj.name, node.arg)) - return filter(None, parsed_names) + return [name for name in parsed_names if name] diff --git a/sandbox/grist/column.py b/sandbox/grist/column.py index dd71d506..f0154a5c 100644 --- a/sandbox/grist/column.py +++ b/sandbox/grist/column.py @@ -2,6 +2,8 @@ import json import types from collections import namedtuple +import six + import depend import objtypes import usertypes @@ -302,7 +304,7 @@ class ChoiceListColumn(BaseColumn): def set(self, row_id, value): # When a JSON string is loaded, set it to a tuple parsed from it. When a list is loaded, # convert to a tuple to keep values immutable. - if isinstance(value, basestring) and value.startswith('['): + if isinstance(value, six.string_types) and value.startswith(u'['): try: value = tuple(json.loads(value)) except Exception: diff --git a/sandbox/grist/csv_patch.py b/sandbox/grist/csv_patch.py index ef04f196..36e21456 100644 --- a/sandbox/grist/csv_patch.py +++ b/sandbox/grist/csv_patch.py @@ -1,5 +1,6 @@ import re import csv +from functools import reduce # Monkey-patch csv.Sniffer class, in which the quote/delimiter detection has silly bugs in the # regexp that it uses. It also seems poorly-implemented in other ways. We can probably do better diff --git a/sandbox/grist/docactions.py b/sandbox/grist/docactions.py index 1185a605..e09fda57 100644 --- a/sandbox/grist/docactions.py +++ b/sandbox/grist/docactions.py @@ -1,6 +1,8 @@ +import six + import actions -import schema import logger +import schema from objtypes import strict_equal log = logger.Logger(__name__, logger.INFO) @@ -15,7 +17,7 @@ class DocActions(object): def AddRecord(self, table_id, row_id, column_values): self.BulkAddRecord( - table_id, [row_id], {key: [val] for key, val in column_values.iteritems()}) + table_id, [row_id], {key: [val] for key, val in six.iteritems(column_values)}) def BulkAddRecord(self, table_id, row_ids, column_values): table = self._engine.tables[table_id] @@ -42,9 +44,9 @@ class DocActions(object): # Collect the undo values, and unset all values in the column (i.e. set to defaults), just to # make sure we don't have stale values hanging around. undo_values = {} - for column in table.all_columns.itervalues(): + for column in six.itervalues(table.all_columns): if not column.is_private() and column.col_id != "id": - col_values = map(column.raw_get, row_ids) + col_values = [column.raw_get(r) for r in row_ids] default = column.getdefault() # If this column had all default values, don't include it into the undo BulkAddRecord. if not all(strict_equal(val, default) for val in col_values): @@ -62,7 +64,7 @@ class DocActions(object): def UpdateRecord(self, table_id, row_id, columns): self.BulkUpdateRecord( - table_id, [row_id], {key: [val] for key, val in columns.iteritems()}) + table_id, [row_id], {key: [val] for key, val in six.iteritems(columns)}) def BulkUpdateRecord(self, table_id, row_ids, columns): table = self._engine.tables[table_id] @@ -72,9 +74,9 @@ class DocActions(object): # Load the updated values. undo_values = {} - for col_id, values in columns.iteritems(): + for col_id, values in six.iteritems(columns): col = table.get_column(col_id) - undo_values[col_id] = map(col.raw_get, row_ids) + undo_values[col_id] = [col.raw_get(r) for r in row_ids] for (row_id, value) in zip(row_ids, values): col.set(row_id, value) @@ -185,7 +187,7 @@ class DocActions(object): log.info("ModifyColumn called which was a noop") return - undo_col_info = {k: v for k, v in schema.col_to_dict(old, include_id=False).iteritems() + undo_col_info = {k: v for k, v in six.iteritems(schema.col_to_dict(old, include_id=False)) if k in col_info} # Remove the column from the schema, then re-add it, to force creation of a new column object. @@ -249,7 +251,7 @@ class DocActions(object): # Copy over all columns from the old table to the new. new_table = self._engine.tables[new_table_id] - for new_column in new_table.all_columns.itervalues(): + for new_column in six.itervalues(new_table.all_columns): if not new_column.is_private(): new_column.copy_from_column(old_table.get_column(new_column.col_id)) new_table.grow_to_max() # We need to bring formula columns to the right size too. diff --git a/sandbox/grist/docmodel.py b/sandbox/grist/docmodel.py index 8def5561..e2e79ca3 100644 --- a/sandbox/grist/docmodel.py +++ b/sandbox/grist/docmodel.py @@ -7,6 +7,8 @@ It is similar in purpose to DocModel.js on the client side. """ import itertools +import six + import records import usertypes import relabeling @@ -107,7 +109,7 @@ def enhance_model(model_class): extras_class = getattr(MetaTableExtras, model_class.__name__, None) if not extras_class: return - for name, member in extras_class.__dict__.iteritems(): + for name, member in six.iteritems(extras_class.__dict__): if not name.startswith("__"): member.__name__ = name member.is_private = True @@ -238,7 +240,7 @@ class DocModel(object): table_obj = record_set_or_table._table group_by = record_set_or_table._group_by if group_by: - values.update((k, [v] * count) for k, v in group_by.iteritems() if k not in values) + values.update((k, [v] * count) for k, v in six.iteritems(group_by) if k not in values) else: table_obj = record_set_or_table.table @@ -281,14 +283,14 @@ def _unify_col_values(col_values, count): Helper that converts a dict mapping keys to values or lists of values to all lists. Non-list values get turned into lists by repeating them count times. """ - assert all(len(v) == count for v in col_values.itervalues() if isinstance(v, list)) + assert all(len(v) == count for v in six.itervalues(col_values) if isinstance(v, list)) return {k: (v if isinstance(v, list) else [v] * count) - for k, v in col_values.iteritems()} + for k, v in six.iteritems(col_values)} def _get_col_values_count(col_values): """ Helper that returns the length of the first list in among the values of col_values. If none of the values is a list, returns 1. """ - first_list = next((v for v in col_values.itervalues() if isinstance(v, list)), None) + first_list = next((v for v in six.itervalues(col_values) if isinstance(v, list)), None) return len(first_list) if first_list is not None else 1 diff --git a/sandbox/grist/engine.py b/sandbox/grist/engine.py index 3dd4c70e..3010a0c5 100644 --- a/sandbox/grist/engine.py +++ b/sandbox/grist/engine.py @@ -11,6 +11,9 @@ import sys import time import traceback from collections import namedtuple, OrderedDict, Hashable + +import six +from six.moves import zip from sortedcontainers import SortedSet import acl @@ -34,8 +37,9 @@ import repl log = logger.Logger(__name__, logger.INFO) -reload(sys) -sys.setdefaultencoding('utf8') +if six.PY2: + reload(sys) + sys.setdefaultencoding('utf8') class OrderError(Exception): @@ -260,11 +264,11 @@ class Engine(object): table = self.tables[data.table_id] # Clear all columns, whether or not they are present in the data. - for column in table.all_columns.itervalues(): + for column in six.itervalues(table.all_columns): column.clear() # Only load columns that aren't stored. - columns = {col_id: data for (col_id, data) in data.columns.iteritems() + columns = {col_id: data for (col_id, data) in six.iteritems(data.columns) if table.has_column(col_id)} # Add the records. @@ -296,10 +300,10 @@ class Engine(object): table.grow_to_max() # Load the new values. - for col_id, values in column_values.iteritems(): + for col_id, values in six.iteritems(column_values): column = table.get_column(col_id) column.growto(growto_size) - for row_id, value in itertools.izip(row_ids, values): + for row_id, value in zip(row_ids, values): column.set(row_id, value) # Invalidate new records to cause the formula columns to get recomputed. @@ -314,16 +318,16 @@ class Engine(object): query_cols = [] if query: - query_cols = [(table.get_column(col_id), values) for (col_id, values) in query.iteritems()] + query_cols = [(table.get_column(col_id), values) for (col_id, values) in six.iteritems(query)] row_ids = [r for r in table.row_ids if all((c.raw_get(r) in values) for (c, values) in query_cols)] - for c in table.all_columns.itervalues(): + for c in six.itervalues(table.all_columns): # pylint: disable=too-many-boolean-expressions if ((formulas or not c.is_formula()) and (private or not c.is_private()) and c.col_id != "id" and not column.is_virtual_column(c.col_id)): - column_values[c.col_id] = map(c.raw_get, row_ids) + column_values[c.col_id] = [c.raw_get(r) for r in row_ids] return actions.TableData(table_id, row_ids, column_values) @@ -355,8 +359,11 @@ class Engine(object): """ schema_actions = schema.schema_create_actions() table_actions = [_get_table_actions(table) for table in self.docmodel.tables.all] - record_actions = [self._get_record_actions(table_id) for (table_id,t) in self.tables.iteritems() - if t.next_row_id() > 1] + record_actions = [ + self._get_record_actions(table_id) + for (table_id,t) in six.iteritems(self.tables) + if t.next_row_id() > 1 + ] return schema_actions + table_actions + record_actions # Returns a BulkAddRecord action which can be used to add the currently existing data to an empty @@ -414,10 +421,10 @@ class Engine(object): meta_tables = self.fetch_table('_grist_Tables') meta_columns = self.fetch_table('_grist_Tables_column') gen_schema = schema.build_schema(meta_tables, meta_columns) - gen_schema_dicts = {k: (t.tableId, dict(t.columns.iteritems())) - for k, t in gen_schema.iteritems()} - cur_schema_dicts = {k: (t.tableId, dict(t.columns.iteritems())) - for k, t in self.schema.iteritems()} + gen_schema_dicts = {k: (t.tableId, dict(t.columns)) + for k, t in six.iteritems(gen_schema)} + cur_schema_dicts = {k: (t.tableId, dict(t.columns)) + for k, t in six.iteritems(self.schema)} if cur_schema_dicts != gen_schema_dicts: import pprint import difflib @@ -448,7 +455,7 @@ class Engine(object): def dump_recompute_map(self): log.debug("Recompute map (%d nodes):" % len(self.recompute_map)) - for node, dirty_rows in self.recompute_map.iteritems(): + for node, dirty_rows in six.iteritems(self.recompute_map): log.debug(" Node %s: %s" % (node, dirty_rows)) @contextlib.contextmanager @@ -507,7 +514,7 @@ class Engine(object): Called at end of _bring_all_up_to_date or _bring_lookups_up_to_date. Issues actions for any accumulated cell changes. """ - for node, changes in self._changes_map.iteritems(): + for node, changes in six.iteritems(self._changes_map): table = self.tables[node.table_id] col = table.get_column(node.col_id) # If there are changes, save them in out_actions. @@ -876,7 +883,7 @@ class Engine(object): table = self.tables[action.table_id] new_values = {} extra_actions = [] - for col_id, values in column_values.iteritems(): + for col_id, values in six.iteritems(column_values): col_obj = table.get_column(col_id) values = [col_obj.convert(val) for val in values] @@ -895,7 +902,7 @@ class Engine(object): # above does it for columns explicitly mentioned; this section does it for the other # columns, using their default values as input to prepare_new_values(). ignore_data = isinstance(action, actions.ReplaceTableData) - for col_id, col_obj in table.all_columns.iteritems(): + for col_id, col_obj in six.iteritems(table.all_columns): if col_id in column_values or column.is_virtual_column(col_id) or col_obj.is_formula(): continue defaults = [col_obj.getdefault() for r in row_ids] @@ -922,7 +929,7 @@ class Engine(object): table = self.tables[action.table_id] # Collect for each column the Column object and a list of new values. - cols = [(table.get_column(col_id), values) for (col_id, values) in column_values.iteritems()] + cols = [(table.get_column(col_id), values) for (col_id, values) in six.iteritems(column_values)] # In comparisons below, we rely here on Python's "==" operator to check for equality. After a # type conversion, it may compare the new type to the old, e.g. 1 == 1.0 == True. It's @@ -988,18 +995,18 @@ class Engine(object): old_tables = self.tables self.tables = {} - for table_id, user_table in self.gencode.usercode.__dict__.iteritems(): + for table_id, user_table in six.iteritems(self.gencode.usercode.__dict__): if isinstance(user_table, table_module.UserTable): self.tables[table_id] = (old_tables.get(table_id) or table_module.Table(table_id, self)) # Now update the table model for each table, and tie it to its UserTable object. - for table_id, table in self.tables.iteritems(): + for table_id, table in six.iteritems(self.tables): user_table = getattr(self.gencode.usercode, table_id) self._update_table_model(table, user_table) user_table._set_table_impl(table) # For any tables that are gone, use self._update_table_model to clean them up. - for table_id, table in old_tables.iteritems(): + for table_id, table in six.iteritems(old_tables): if table_id not in self.tables: self._update_table_model(table, None) self._repl.locals.pop(table_id, None) @@ -1032,8 +1039,8 @@ class Engine(object): table._rebuild_model(user_table) new_columns = table.all_columns - added_col_ids = new_columns.viewkeys() - old_columns.viewkeys() - deleted_col_ids = old_columns.viewkeys() - new_columns.viewkeys() + added_col_ids = six.viewkeys(new_columns) - six.viewkeys(old_columns) + deleted_col_ids = six.viewkeys(old_columns) - six.viewkeys(new_columns) # Invalidate the columns that got added and anything that depends on them. if added_col_ids: @@ -1101,7 +1108,7 @@ class Engine(object): if self._schema_updated: self.assert_schema_consistent() - except Exception, e: + except Exception as e: # Save full exception info, so that we can rethrow accurately even if undo also fails. exc_info = sys.exc_info() # If we get an exception, we should revert all changes applied so far, to keep things @@ -1249,7 +1256,7 @@ class Engine(object): (len_calc, len_stored, len_undo, len_ret) = checkpoint undo_actions = self.out_actions.undo[len_undo:] log.info("Reverting %d doc actions" % len(undo_actions)) - self.user_actions.ApplyUndoActions(map(actions.get_action_repr, undo_actions)) + self.user_actions.ApplyUndoActions([actions.get_action_repr(a) for a in undo_actions]) del self.out_actions.calc[len_calc:] del self.out_actions.stored[len_stored:] del self.out_actions.direct[len_stored:] diff --git a/sandbox/grist/functions/__init__.py b/sandbox/grist/functions/__init__.py index 485f4dda..007dffe0 100644 --- a/sandbox/grist/functions/__init__.py +++ b/sandbox/grist/functions/__init__.py @@ -1,12 +1,12 @@ # pylint: disable=wildcard-import -from date import * -from info import * -from logical import * -from lookup import * -from math import * -from stats import * -from text import * -from schedule import * +from .date import * +from .info import * +from .logical import * +from .lookup import * +from .math import * +from .stats import * +from .text import * +from .schedule import * # Export all uppercase names, for use with `from functions import *`. __all__ = [k for k in dir() if not k.startswith('_') and k.isupper()] diff --git a/sandbox/grist/functions/date.py b/sandbox/grist/functions/date.py index bcdd73fc..a2ba2692 100644 --- a/sandbox/grist/functions/date.py +++ b/sandbox/grist/functions/date.py @@ -1,6 +1,8 @@ import calendar import datetime import dateutil.parser +import six + import moment import docmodel @@ -16,7 +18,7 @@ def _make_datetime(value): return datetime.datetime.combine(value, datetime.time()) elif isinstance(value, datetime.time): return datetime.datetime.combine(datetime.date.today(), value) - elif isinstance(value, basestring): + elif isinstance(value, six.string_types): return dateutil.parser.parse(value) else: raise ValueError('Invalid date %r' % (value,)) diff --git a/sandbox/grist/functions/info.py b/sandbox/grist/functions/info.py index 80ecf168..7584f3a6 100644 --- a/sandbox/grist/functions/info.py +++ b/sandbox/grist/functions/info.py @@ -7,6 +7,8 @@ import math import numbers import re +import six + import column from functions import date # pylint: disable=import-error from functions.unimplemented import unimplemented @@ -217,7 +219,7 @@ def ISTEXT(value): >>> ISTEXT(datetime.date(2011, 1, 1)) False """ - return isinstance(value, (basestring, AltText)) + return isinstance(value, (six.string_types, AltText)) # Regexp for matching email. See ISEMAIL for justification. diff --git a/sandbox/grist/functions/logical.py b/sandbox/grist/functions/logical.py index 966278b2..741e781d 100644 --- a/sandbox/grist/functions/logical.py +++ b/sandbox/grist/functions/logical.py @@ -1,4 +1,4 @@ -from info import lazy_value_or_error, is_error +from .info import lazy_value_or_error, is_error from usertypes import AltText # pylint: disable=unused-import,import-error @@ -63,7 +63,7 @@ def IF(logical_expression, value_if_true, value_if_false): 0.0 More tests: - >>> IF(True, lambda: (1/0), lambda: (17)) + >>> IF(True, lambda: (1/0), lambda: (17)) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... ZeroDivisionError: integer division or modulo by zero diff --git a/sandbox/grist/functions/lookup.py b/sandbox/grist/functions/lookup.py index 31dd5c57..76492d02 100644 --- a/sandbox/grist/functions/lookup.py +++ b/sandbox/grist/functions/lookup.py @@ -1,9 +1,10 @@ # pylint: disable=redefined-builtin, line-too-long from collections import OrderedDict import os -from urllib import urlencode -import urlparse -from unimplemented import unimplemented + +import six +from six.moves import urllib_parse +from .unimplemented import unimplemented @unimplemented def ADDRESS(row, column, absolute_relative_mode, use_a1_notation, sheet): @@ -112,16 +113,16 @@ def SELF_HYPERLINK(label=None, page=None, **kwargs): if page: txt += "/p/{}".format(page) if kwargs: - parts = list(urlparse.urlparse(txt)) - query = OrderedDict(urlparse.parse_qsl(parts[4])) - for [key, value] in kwargs.iteritems(): + parts = list(urllib_parse.urlparse(txt)) + query = OrderedDict(urllib_parse.parse_qsl(parts[4])) + for [key, value] in sorted(six.iteritems(kwargs)): key_parts = key.split('LinkKey_') if len(key_parts) == 2 and key_parts[0] == '': query[key_parts[1] + '_'] = value else: raise TypeError("unexpected keyword argument '{}' (not of form LinkKey_NAME)".format(key)) - parts[4] = urlencode(query) - txt = urlparse.urlunparse(parts) + parts[4] = urllib_parse.urlencode(query) + txt = urllib_parse.urlunparse(parts) if label: txt = "{} {}".format(label, txt) return txt diff --git a/sandbox/grist/functions/math.py b/sandbox/grist/functions/math.py index fc7e2dff..09e5ed59 100644 --- a/sandbox/grist/functions/math.py +++ b/sandbox/grist/functions/math.py @@ -1,12 +1,15 @@ # pylint: disable=unused-argument from __future__ import absolute_import -import itertools import math as _math import operator import os import random import uuid +from functools import reduce + +from six.moves import zip, xrange +import six from functions.info import ISNUMBER, ISLOGICAL from functions.unimplemented import unimplemented @@ -358,7 +361,7 @@ def INT(value): return int(_math.floor(value)) def _lcm(a, b): - return a * b / _gcd(a, b) + return a * b // _gcd(a, b) def LCM(value1, *more_values): """ @@ -790,7 +793,7 @@ def SUMPRODUCT(array1, *more_arrays): >>> SUMPRODUCT([-0.25, -0.25], [-2, -2], [-3, -3]) -3.0 """ - return sum(reduce(operator.mul, values) for values in itertools.izip(array1, *more_arrays)) + return sum(reduce(operator.mul, values) for values in zip(array1, *more_arrays)) @unimplemented def SUMSQ(value1, value2): @@ -842,4 +845,7 @@ def TRUNC(value, places=0): def UUID(): """Generate a random UUID-formatted string identifier.""" - return str(uuid.UUID(bytes=[chr(random.randrange(0, 256)) for _ in xrange(0, 16)], version=4)) + if six.PY2: + return str(uuid.UUID(bytes=[chr(random.randrange(0, 256)) for _ in xrange(0, 16)], version=4)) + else: + return str(uuid.UUID(bytes=bytes([random.randrange(0, 256) for _ in range(0, 16)]), version=4)) diff --git a/sandbox/grist/functions/schedule.py b/sandbox/grist/functions/schedule.py index 30aa398f..b66a187a 100644 --- a/sandbox/grist/functions/schedule.py +++ b/sandbox/grist/functions/schedule.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta import re -from date import DATEADD, NOW, DTIME +from .date import DATEADD, NOW, DTIME from moment_parse import MONTH_NAMES, DAY_NAMES # Limit exports to schedule, so that upper-case constants like MONTH_NAMES, DAY_NAMES don't end up diff --git a/sandbox/grist/functions/stats.py b/sandbox/grist/functions/stats.py index a4f56f8a..08bcf799 100644 --- a/sandbox/grist/functions/stats.py +++ b/sandbox/grist/functions/stats.py @@ -1,9 +1,9 @@ # pylint: disable=redefined-builtin, line-too-long, unused-argument -from math import _chain, _chain_numeric, _chain_numeric_a -from info import ISNUMBER, ISLOGICAL -from date import DATE # pylint: disable=unused-import -from unimplemented import unimplemented +from .math import _chain, _chain_numeric, _chain_numeric_a +from .info import ISNUMBER, ISLOGICAL +from .date import DATE # pylint: disable=unused-import +from .unimplemented import unimplemented def _average(iterable): total, count = 0.0, 0 @@ -326,7 +326,7 @@ def MEDIAN(value, *more_values): 3 >>> MEDIAN(3, 5, 1, 4, 2) 3 - >>> MEDIAN(xrange(10)) + >>> MEDIAN(range(10)) 4.5 >>> MEDIAN("Hello", "123", DATE(2015, 1, 1), 12.3) 12.3 @@ -340,9 +340,9 @@ def MEDIAN(value, *more_values): raise ValueError("MEDIAN requires at least one number") count = len(values) if count % 2 == 0: - return (values[count / 2 - 1] + values[count / 2]) / 2.0 + return (values[count // 2 - 1] + values[count // 2]) / 2.0 else: - return values[(count - 1) / 2] + return values[(count - 1) // 2] def MIN(value, *more_values): diff --git a/sandbox/grist/functions/test_schedule.py b/sandbox/grist/functions/test_schedule.py index f75769d6..c1a9ffe6 100644 --- a/sandbox/grist/functions/test_schedule.py +++ b/sandbox/grist/functions/test_schedule.py @@ -4,7 +4,7 @@ import timeit import unittest import moment -import schedule +from . import schedule from functions.date import DTIME from functions import date as _date @@ -68,7 +68,7 @@ class TestSchedule(unittest.TestCase): self.assertDate(RDU(DT("2018-09-04 14:38:11"), "seconds"), "2018-09-04 14:38:11") self.assertDate(RDU(DT("2018-09-04 14:38:11") - TICK, "seconds"), "2018-09-04 14:38:10") - with self.assertRaisesRegexp(ValueError, r"Invalid unit inches"): + with self.assertRaisesRegex(ValueError, r"Invalid unit inches"): RDU(DT("2018-09-04 14:38:11"), "inches") def test_round_down_to_unit_tz(self): @@ -99,11 +99,11 @@ class TestSchedule(unittest.TestCase): self.assertEqual(schedule._parse_interval("25-months"), (25, "months")) self.assertEqual(schedule._parse_interval("3-day"), (3, "days")) self.assertEqual(schedule._parse_interval("2-hour"), (2, "hours")) - with self.assertRaisesRegexp(ValueError, "Not a valid interval"): + with self.assertRaisesRegex(ValueError, "Not a valid interval"): schedule._parse_interval("1Year") - with self.assertRaisesRegexp(ValueError, "Not a valid interval"): + with self.assertRaisesRegex(ValueError, "Not a valid interval"): schedule._parse_interval("1y") - with self.assertRaisesRegexp(ValueError, "Unknown unit"): + with self.assertRaisesRegex(ValueError, "Unknown unit"): schedule._parse_interval("1-daily") def test_parse_slot(self): @@ -145,41 +145,41 @@ class TestSchedule(unittest.TestCase): def test_parse_slot_errors(self): # Test failures with duplicate units - with self.assertRaisesRegexp(ValueError, 'Duplicate unit'): + with self.assertRaisesRegex(ValueError, 'Duplicate unit'): schedule._parse_slot('+1d +2d', 'weeks') - with self.assertRaisesRegexp(ValueError, 'Duplicate unit'): + with self.assertRaisesRegex(ValueError, 'Duplicate unit'): schedule._parse_slot('9:30am +2H', 'days') - with self.assertRaisesRegexp(ValueError, 'Duplicate unit'): + with self.assertRaisesRegex(ValueError, 'Duplicate unit'): schedule._parse_slot('/15 +1d', 'months') - with self.assertRaisesRegexp(ValueError, 'Duplicate unit'): + with self.assertRaisesRegex(ValueError, 'Duplicate unit'): schedule._parse_slot('Feb-1 12:30pm +20M', 'years') # Test failures with improper slot types - with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'): + with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'): schedule._parse_slot('Feb-1', 'weeks') - with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'): + with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'): schedule._parse_slot('Monday', 'months') - with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'): + with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'): schedule._parse_slot('4/15', 'hours') - with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'): + with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'): schedule._parse_slot('/1', 'years') # Test failures with outright invalid slot syntax. - with self.assertRaisesRegexp(ValueError, 'Invalid slot'): + with self.assertRaisesRegex(ValueError, 'Invalid slot'): schedule._parse_slot('Feb:1', 'weeks') - with self.assertRaisesRegexp(ValueError, 'Invalid slot'): + with self.assertRaisesRegex(ValueError, 'Invalid slot'): schedule._parse_slot('/1d', 'months') - with self.assertRaisesRegexp(ValueError, 'Invalid slot'): + with self.assertRaisesRegex(ValueError, 'Invalid slot'): schedule._parse_slot('10', 'hours') - with self.assertRaisesRegexp(ValueError, 'Invalid slot'): + with self.assertRaisesRegex(ValueError, 'Invalid slot'): schedule._parse_slot('H1', 'years') # Test failures with unknown values - with self.assertRaisesRegexp(ValueError, 'Unknown month'): + with self.assertRaisesRegex(ValueError, 'Unknown month'): schedule._parse_slot('februarium-1', 'years') - with self.assertRaisesRegexp(ValueError, 'Unknown day of the week'): + with self.assertRaisesRegex(ValueError, 'Unknown day of the week'): schedule._parse_slot('snu', 'weeks') - with self.assertRaisesRegexp(ValueError, 'Unknown unit'): + with self.assertRaisesRegex(ValueError, 'Unknown unit'): schedule._parse_slot('+1t', 'hours') def test_schedule(self): @@ -250,14 +250,14 @@ from datetime import datetime ] self.assertEqual(timing_schedule_full(), expected_result) t = min(timeit.repeat(stmt="t.timing_schedule_full()", setup=setup, number=N, repeat=3)) - print "\n*** SCHEDULE call with 4 points: %.2f us" % (t * 1000000 / N) + print("\n*** SCHEDULE call with 4 points: %.2f us" % (t * 1000000 / N)) t = min(timeit.repeat(stmt="t.timing_schedule_init()", setup=setup, number=N, repeat=3)) - print "*** Schedule constructor: %.2f us" % (t * 1000000 / N) + print("*** Schedule constructor: %.2f us" % (t * 1000000 / N)) self.assertEqual(timing_schedule_series(), expected_result) t = min(timeit.repeat(stmt="t.timing_schedule_series()", setup=setup, number=N, repeat=3)) - print "*** Schedule series with 4 points: %.2f us" % (t * 1000000 / N) + print("*** Schedule series with 4 points: %.2f us" % (t * 1000000 / N)) def timing_schedule_full(): return list(schedule.SCHEDULE("weekly: Mo 10:30am, We 10:30pm", diff --git a/sandbox/grist/functions/text.py b/sandbox/grist/functions/text.py index 83edd13a..ffdc6838 100644 --- a/sandbox/grist/functions/text.py +++ b/sandbox/grist/functions/text.py @@ -5,7 +5,11 @@ import dateutil.parser import numbers import re -from unimplemented import unimplemented +import six +from six import unichr +from six.moves import xrange + +from .unimplemented import unimplemented from usertypes import AltText # pylint: disable=import-error def CHAR(table_number): @@ -478,7 +482,7 @@ def SUBSTITUTE(text, old_text, new_text, instance_num=None): if not old_text: return text - if not isinstance(new_text, basestring): + if not isinstance(new_text, six.string_types): new_text = str(new_text) if instance_num is None: diff --git a/sandbox/grist/gencode.py b/sandbox/grist/gencode.py index 0ce113f5..ac4f8644 100644 --- a/sandbox/grist/gencode.py +++ b/sandbox/grist/gencode.py @@ -19,6 +19,8 @@ import re import imp from collections import OrderedDict +import six + import codebuilder from column import is_visible_column import summary @@ -123,7 +125,7 @@ class GenCode(object): source_table_id = summary.decode_summary_table_name(table_id) # Sort columns by "isFormula" to output all data columns before all formula columns. - columns = sorted(table_info.columns.itervalues(), key=lambda c: c.isFormula) + columns = sorted(six.itervalues(table_info.columns), key=lambda c: c.isFormula) if filter_for_user: columns = [c for c in columns if is_visible_column(c.colId)] parts = ["@grist.UserTable\nclass %s:\n" % table_id] @@ -136,9 +138,9 @@ class GenCode(object): if summary_tables: # Include summary formulas, for the user's information. formulas = OrderedDict((c.colId, c) for s in summary_tables - for c in s.columns.itervalues() if c.isFormula) + for c in six.itervalues(s.columns) if c.isFormula) parts.append(indent(textbuilder.Text("\nclass _Summary:\n"))) - for col_info in formulas.itervalues(): + for col_info in six.itervalues(formulas): parts.append(indent(self._make_field(col_info, table_id), levels=2)) return textbuilder.Combiner(parts) @@ -147,7 +149,7 @@ class GenCode(object): """Regenerates the code text and usercode module from upated document schema.""" # Collect summary tables to group them by source table. summary_tables = {} - for table_info in schema.itervalues(): + for table_info in six.itervalues(schema): source_table_id = summary.decode_summary_table_name(table_info.tableId) if source_table_id: summary_tables.setdefault(source_table_id, []).append(table_info) @@ -156,7 +158,7 @@ class GenCode(object): "from functions import * # global uppercase functions\n" + "import datetime, math, re # modules commonly needed in formulas\n"] userparts = fullparts[:] - for table_info in schema.itervalues(): + for table_info in six.itervalues(schema): fullparts.append("\n\n") fullparts.append(self._make_table_model(table_info, summary_tables.get(table_info.tableId))) if not _is_special_table(table_info.tableId): @@ -191,5 +193,5 @@ def _is_special_table(table_id): def exec_module_text(module_text): # pylint: disable=exec-used mod = imp.new_module("usercode") - exec module_text in mod.__dict__ + exec(module_text, mod.__dict__) return mod diff --git a/sandbox/grist/gpath.py b/sandbox/grist/gpath.py index 9e6de31b..56cbcd15 100644 --- a/sandbox/grist/gpath.py +++ b/sandbox/grist/gpath.py @@ -1,3 +1,6 @@ +from six.moves import xrange + + def _is_array(obj): return isinstance(obj, list) diff --git a/sandbox/grist/import_actions.py b/sandbox/grist/import_actions.py index 5274599c..e8b680ae 100644 --- a/sandbox/grist/import_actions.py +++ b/sandbox/grist/import_actions.py @@ -287,7 +287,8 @@ class ImportActions(object): src_col_id = _import_transform_col_prefix + curr_col["colId"] log.debug("Copying from: " + src_col_id) - column_data[curr_col["colId"]] = map(hidden_table.get_column(src_col_id).raw_get, row_ids) + src_col = hidden_table.get_column(src_col_id) + column_data[curr_col["colId"]] = [src_col.raw_get(r) for r in row_ids] # ========= Cleanup, Prepare new table (if needed), insert data diff --git a/sandbox/grist/imports/test_messytables.py b/sandbox/grist/imports/test_messytables.py index fe25db5d..d36ce731 100644 --- a/sandbox/grist/imports/test_messytables.py +++ b/sandbox/grist/imports/test_messytables.py @@ -8,7 +8,7 @@ class TestMessyTables(unittest.TestCase): def test_any_tableset(self): path = os.path.join(os.path.dirname(__file__), "fixtures", "nyc_schools_progress_report_ec_2013.xlsx") - with open(path, "r") as f: + with open(path, "rb") as f: table_set = messytables.any.any_tableset(f, extension=os.path.splitext(path)[1]) self.assertIsInstance(table_set, messytables.XLSTableSet) diff --git a/sandbox/grist/lookup.py b/sandbox/grist/lookup.py index f2d6046a..5050ccf5 100644 --- a/sandbox/grist/lookup.py +++ b/sandbox/grist/lookup.py @@ -1,3 +1,5 @@ +import six + import column import depend import records @@ -85,7 +87,7 @@ class LookupMapColumn(column.BaseColumn): def _invalidate_affected(self, affected_keys): # For each known relation, figure out which referring rows are affected, and invalidate them. # The engine will notice that there have been more invalidations, and recompute things again. - for node, rel in self._lookup_relations.iteritems(): + for node, rel in six.iteritems(self._lookup_relations): affected_rows = rel.get_affected_rows_by_keys(affected_keys) self._engine.invalidate_records(node.table_id, affected_rows, col_ids=(node.col_id,)) diff --git a/sandbox/grist/main.py b/sandbox/grist/main.py index 9a01652d..3be8007a 100644 --- a/sandbox/grist/main.py +++ b/sandbox/grist/main.py @@ -9,6 +9,8 @@ sys.path.append('thirdparty') import marshal import functools +import six + from acl_formula import parse_acl_formula import actions import sandbox @@ -86,7 +88,7 @@ def main(): @export def fetch_meta_tables(formulas=True): return {table_id: actions.get_action_repr(table_data) - for (table_id, table_data) in eng.fetch_meta_tables(formulas).iteritems()} + for (table_id, table_data) in six.iteritems(eng.fetch_meta_tables(formulas))} @export def load_meta_tables(meta_tables, meta_columns): @@ -100,8 +102,8 @@ def main(): @export def create_migrations(all_tables, metadata_only=False): doc_actions = migrations.create_migrations( - {t: table_data_from_db(t, data) for t, data in all_tables.iteritems()}, metadata_only) - return map(actions.get_action_repr, doc_actions) + {t: table_data_from_db(t, data) for t, data in six.iteritems(all_tables)}, metadata_only) + return [actions.get_action_repr(action) for action in doc_actions] @export def get_version(): diff --git a/sandbox/grist/migrations.py b/sandbox/grist/migrations.py index 5589bb0d..18d6bbc7 100644 --- a/sandbox/grist/migrations.py +++ b/sandbox/grist/migrations.py @@ -1,6 +1,9 @@ import json import re +import six +from six.moves import xrange + import actions import identifiers import schema @@ -56,12 +59,12 @@ def create_migrations(all_tables, metadata_only=False): user_schema = schema.build_schema(all_tables['_grist_Tables'], all_tables['_grist_Tables_column'], include_builtin=False) - for t in user_schema.itervalues(): + for t in six.itervalues(user_schema): tdset.apply_doc_action(actions.AddTable(t.tableId, schema.cols_to_dict_list(t.columns))) # For each old table/column, construct an AddTable action using the current schema. new_schema = {a.table_id: a for a in schema.schema_create_actions()} - for table_id, data in sorted(all_tables.iteritems()): + for table_id, data in sorted(six.iteritems(all_tables)): # User tables should already be in tdset; the rest must be metadata tables. # (If metadata_only is true, there is simply nothing to skip here.) if table_id not in tdset.all_tables: @@ -94,7 +97,7 @@ def get_last_migration_version(): """ Returns the last schema version number for which we have a migration defined. """ - return max(all_migrations.iterkeys()) + return max(all_migrations) def migration(schema_version, need_all_tables=False): """ @@ -330,16 +333,16 @@ def migration7(tdset): # - It doesn't fix types of Reference columns that refer to old-style summary tables # (if the user created some such columns manually). - doc_actions = filter(None, [ + doc_actions = [action for action in [ maybe_add_column(tdset, '_grist_Tables', 'summarySourceTable', 'Ref:_grist_Tables'), maybe_add_column(tdset, '_grist_Tables_column', 'summarySourceCol', 'Ref:_grist_Tables_column') - ]) + ] if action] # Maps tableRef to Table object. tables_map = {t.id: t for t in actions.transpose_bulk_action(tdset.all_tables['_grist_Tables'])} # Maps tableName to tableRef - table_name_to_ref = {t.tableId: t.id for t in tables_map.itervalues()} + table_name_to_ref = {t.tableId: t.id for t in six.itervalues(tables_map)} # List of Column objects columns = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables_column'])) @@ -362,14 +365,14 @@ def migration7(tdset): # Summary tables used to be named as "Summary___". This regular # expression parses that. summary_re = re.compile(r'^Summary_(\w+?)((?:_\d+)*)$') - for t in tables_map.itervalues(): + for t in six.itervalues(tables_map): m = summary_re.match(t.tableId) if not m or m.group(1) not in table_name_to_ref: continue # We have a valid summary table. source_table_name = m.group(1) source_table_ref = table_name_to_ref[source_table_name] - groupby_colrefs = map(int, m.group(2).strip("_").split("_")) + groupby_colrefs = [int(x) for x in m.group(2).strip("_").split("_")] # Prepare a new-style name for the summary table. Be sure not to conflict with existing tables # or with each other (i.e. don't rename multiple tables to the same name). new_name = summary.encode_summary_table_name(source_table_name) diff --git a/sandbox/grist/moment.py b/sandbox/grist/moment.py index c20f9196..3e57d15b 100644 --- a/sandbox/grist/moment.py +++ b/sandbox/grist/moment.py @@ -3,10 +3,11 @@ from collections import namedtuple import marshal from time import time import bisect -import itertools import os import moment_parse import iso8601 +import six +from six.moves import zip # This is prepared by sandbox/install_tz.py ZoneRecord = namedtuple("ZoneRecord", ("name", "abbrs", "offsets", "untils")) @@ -92,7 +93,7 @@ class tz(object): self._tzinfo = tzinfo(zonelabel) if isinstance(dt, datetime): timestamp = dt_to_ts(dt.replace(tzinfo=self._tzinfo)) * 1000 - elif isinstance(dt, (float, int, long)): + elif isinstance(dt, (float, six.integer_types)): timestamp = dt else: raise TypeError("'dt' should be a datetime object or a numeric type") @@ -181,7 +182,7 @@ class Zone(object): # "Until" times adjusted by the corresponding offsets. These are used in translating from # datetime to absolute timestamp. self.offset_untils = [until - offset * 60000 for (until, offset) in - itertools.izip(self.untils, self.offsets)] + zip(self.untils, self.offsets)] # Cache of TzInfo objects for this Zone, used by get_tzinfo(). There could be multiple TzInfo # objects, one for each possible offset, but their behavior only differs for ambiguous time. self._tzinfo = {} diff --git a/sandbox/grist/objtypes.py b/sandbox/grist/objtypes.py index aed859ac..11739643 100644 --- a/sandbox/grist/objtypes.py +++ b/sandbox/grist/objtypes.py @@ -17,6 +17,7 @@ from math import isnan import moment import records +import six class UnmarshallableError(ValueError): @@ -186,9 +187,9 @@ def encode_object(value): # Represent RecordSet (e.g. result of lookupRecords) in the same way as a RecordList. return ['L'] + [encode_object(int(item)) for item in value] elif isinstance(value, dict): - if not all(isinstance(key, basestring) for key in value): + if not all(isinstance(key, six.string_types) for key in value): raise UnmarshallableError("Dict with non-string keys") - return ['O', {key: encode_object(val) for key, val in value.iteritems()}] + return ['O', {key: encode_object(val) for key, val in six.iteritems(value)}] elif value == _pending_sentinel: return ['P'] elif value == _censored_sentinel: @@ -230,7 +231,7 @@ def decode_object(value): elif code == 'L': return [decode_object(item) for item in args] elif code == 'O': - return {decode_object(key): decode_object(val) for key, val in args[0].iteritems()} + return {decode_object(key): decode_object(val) for key, val in six.iteritems(args[0])} elif code == 'P': return _pending_sentinel elif code == 'C': diff --git a/sandbox/grist/records.py b/sandbox/grist/records.py index 08c885c5..6ceb99f1 100644 --- a/sandbox/grist/records.py +++ b/sandbox/grist/records.py @@ -97,6 +97,8 @@ class Record(object): def __nonzero__(self): return bool(self._row_id) + __bool__ = __nonzero__ + def __repr__(self): return "%s[%s]" % (self._table.table_id, self._row_id) @@ -144,6 +146,8 @@ class RecordSet(object): def __nonzero__(self): return bool(self._row_ids) + __bool__ = __nonzero__ + def __eq__(self, other): return (isinstance(other, RecordSet) and (self._table, self._row_ids, self._group_by, self._sort_by) == diff --git a/sandbox/grist/relabeling.py b/sandbox/grist/relabeling.py index 00660cb3..027391d1 100644 --- a/sandbox/grist/relabeling.py +++ b/sandbox/grist/relabeling.py @@ -44,6 +44,7 @@ import itertools import math import struct +from six.moves import zip, xrange from sortedcontainers import SortedList, SortedListWithKey @@ -79,7 +80,7 @@ def prepare_inserts_dumb(sortedlist, keys): ins_groups.append((len(sortedlist), 0)) for index, ins_count in ins_groups: adj_count = index - prev_index - adjustments.extend(itertools.izip(xrange(prev_index, index), + adjustments.extend(zip(xrange(prev_index, index), frange_from(next_key, adj_count))) next_key += adj_count insertions.extend(frange_from(next_key, ins_count)) @@ -233,7 +234,7 @@ class ListWithAdjustments(object): prev_keys.sort() new_keys = get_range(new_begin_key, new_end_key, count) - for (old_key, is_insert, i), new_key in itertools.izip(prev_keys, new_keys): + for (old_key, is_insert, i), new_key in zip(prev_keys, new_keys): if is_insert: self._insertions.remove(old_key) self._insertions.add(new_key) @@ -303,7 +304,7 @@ def all_distinct(iterable): """ a, b = itertools.tee(iterable) next(b, None) - return all(x != y for x, y in itertools.izip(a, b)) + return all(x != y for x, y in zip(a, b)) def range_around_float(x, i): diff --git a/sandbox/grist/repl.py b/sandbox/grist/repl.py index 6a9effa9..76eb8fb4 100644 --- a/sandbox/grist/repl.py +++ b/sandbox/grist/repl.py @@ -57,7 +57,7 @@ class REPLInterpreter(code.InteractiveInterpreter): # like get/set attr and have that hurt us sys.stdout = user_output sys.stderr = user_output - exec code in self.locals + exec(code, self.locals) except: # bare except to catch absolutely all things the user can throw self.showtraceback() diff --git a/sandbox/grist/runtests.py b/sandbox/grist/runtests.py index 7f38c075..5b8ceb94 100644 --- a/sandbox/grist/runtests.py +++ b/sandbox/grist/runtests.py @@ -14,6 +14,8 @@ import sys import unittest sys.path.append('/thirdparty') +import six + def main(): # Change to the directory of this file (/grist in sandbox), to discover everything under it. os.chdir(os.path.dirname(__file__)) @@ -23,7 +25,9 @@ def main(): if "--xunit" in argv: import xmlrunner argv.remove("--xunit") - utf8_stdout = codecs.getwriter('utf8')(sys.stdout) + utf8_stdout = sys.stdout + if six.PY2: + utf8_stdout = codecs.getwriter('utf8')(utf8_stdout) test_runner = xmlrunner.XMLTestRunner(stream=utf8_stdout) if all(arg.startswith("-") for arg in argv[1:]): diff --git a/sandbox/grist/sandbox.py b/sandbox/grist/sandbox.py index ddf316c8..65d9c1e4 100644 --- a/sandbox/grist/sandbox.py +++ b/sandbox/grist/sandbox.py @@ -38,8 +38,8 @@ class Sandbox(object): def __init__(self): self._functions = {} - self._external_input = os.fdopen(3, "r", 64*1024) - self._external_output = os.fdopen(4, "w", 64*1024) + self._external_input = os.fdopen(3, "rb", 64*1024) + self._external_output = os.fdopen(4, "wb", 64*1024) def _send_to_js(self, msgCode, msgBody): # (Note that marshal version 2 is the default; we specify it explicitly for clarity. The diff --git a/sandbox/grist/schema.py b/sandbox/grist/schema.py index df1c5da0..60403e11 100644 --- a/sandbox/grist/schema.py +++ b/sandbox/grist/schema.py @@ -10,6 +10,9 @@ Before changing this file, please review: import itertools from collections import OrderedDict, namedtuple + +import six + import actions SCHEMA_VERSION = 21 @@ -310,7 +313,7 @@ def cols_to_dict_list(cols): def clone_schema(schema): return OrderedDict((t, SchemaTable(s.tableId, s.columns.copy())) - for (t, s) in schema.iteritems()) + for (t, s) in six.iteritems(schema)) def build_schema(meta_tables, meta_columns, include_builtin=True): """ diff --git a/sandbox/grist/summary.py b/sandbox/grist/summary.py index f0f4649b..88e02bce 100644 --- a/sandbox/grist/summary.py +++ b/sandbox/grist/summary.py @@ -1,6 +1,9 @@ from collections import namedtuple import json import re + +import six + import logger log = logger.Logger(__name__, logger.INFO) @@ -17,7 +20,7 @@ def _make_col_info(col=None, **values): def _get_colinfo_dict(col_info, with_id=False): """Return a dict suitable to use with AddColumn or AddTable (when with_id=True) actions.""" - col_values = {k: v for k, v in col_info._asdict().iteritems() if v is not None and k != 'colId'} + col_values = {k: v for k, v in six.iteritems(col_info._asdict()) if v is not None and k != 'colId'} if with_id: col_values['id'] = col_info.colId return col_values @@ -78,9 +81,10 @@ def _update_sort_spec(sort_spec, old_table, new_table): try: old_sort_spec = json.loads(sort_spec) - new_sort_spec = filter(None, [adjust(col_spec) for col_spec in old_sort_spec]) + new_sort_spec = [adjust(col_spec) for col_spec in old_sort_spec] + new_sort_spec = [col_spec for col_spec in new_sort_spec if col_spec] return json.dumps(new_sort_spec, separators=(',', ':')) - except Exception, e: + except Exception: log.warn("update_summary_section: can't parse sortColRefs JSON; clearing sortColRefs") return '' diff --git a/sandbox/grist/table.py b/sandbox/grist/table.py index 7bfc64a1..2e42636e 100644 --- a/sandbox/grist/table.py +++ b/sandbox/grist/table.py @@ -1,6 +1,9 @@ import collections import types +import six +from six.moves import xrange + import column import depend import docmodel @@ -211,7 +214,7 @@ class Table(object): new_cols['id'] = self._id_column # List of Columns in the same order as they appear in the generated Model definition. - col_items = [c for c in self.Model.__dict__.iteritems() if not c[0].startswith("_")] + col_items = [c for c in six.iteritems(self.Model.__dict__) if not c[0].startswith("_")] col_items.sort(key=lambda c: self._get_sort_order(c[1])) for col_id, col_model in col_items: @@ -219,11 +222,11 @@ class Table(object): new_cols[col_id] = self._create_or_update_col(col_id, col_model, default_func) # Used for auto-completion as a record with correct properties of correct types. - self.sample_record = _make_sample_record(self.table_id, new_cols.itervalues()) + self.sample_record = _make_sample_record(self.table_id, six.itervalues(new_cols)) # Note that we reuse previous special columns like lookup maps, since those not affected by # column changes should stay the same. These get removed when unneeded using other means. - new_cols.update(sorted(self._special_cols.iteritems())) + new_cols.update(sorted(six.iteritems(self._special_cols))) # Set the new columns. self.all_columns = new_cols @@ -289,7 +292,7 @@ class Table(object): """ return ((0, col_model._creation_order) if not isinstance(col_model, types.FunctionType) else - (1, col_model.func_code.co_firstlineno)) + (1, col_model.__code__.co_firstlineno)) def next_row_id(self): """ @@ -302,7 +305,7 @@ class Table(object): Resizes all columns as needed so that all valid row_ids are valid indices into all columns. """ size = self.row_ids.max() + 1 - for col_obj in self.all_columns.itervalues(): + for col_obj in six.itervalues(self.all_columns): col_obj.growto(size) def get_column(self, col_id): @@ -325,7 +328,7 @@ class Table(object): """ # The tuple of keys used determines the LookupMap we need. sort_by = kwargs.pop('sort_by', None) - col_ids = tuple(sorted(kwargs.iterkeys())) + col_ids = tuple(sorted(kwargs)) key = tuple(kwargs[c] for c in col_ids) lookup_map = self._get_lookup_map(col_ids) @@ -383,7 +386,7 @@ class Table(object): # TODO: It should use indices, to avoid linear searching # TODO: It should create dependencies as needed when used from formulas. # TODO: It should return Record instead, for convenience of user formulas - col_values = [(self.all_columns[col_id], value) for (col_id, value) in kwargs.iteritems()] + col_values = [(self.all_columns[col_id], value) for (col_id, value) in six.iteritems(kwargs)] for row_id in self.row_ids: if all(col.raw_get(row_id) == value for col, value in col_values): return row_id @@ -398,7 +401,7 @@ class Table(object): # TODO: It should use indices, to avoid linear searching # TODO: It should create dependencies as needed when used from formulas. # TODO: It should return Record instead, for convenience of user formulas - col_values = [(self.all_columns[col_id], value) for (col_id, value) in kwargs.iteritems()] + col_values = [(self.all_columns[col_id], value) for (col_id, value) in six.iteritems(kwargs)] for row_id in self.row_ids: if all(col.raw_get(row_id) == value for col, value in col_values): yield row_id diff --git a/sandbox/grist/table_data_set.py b/sandbox/grist/table_data_set.py index 58750a17..18a79f12 100644 --- a/sandbox/grist/table_data_set.py +++ b/sandbox/grist/table_data_set.py @@ -1,4 +1,6 @@ -from itertools import izip +from six.moves import zip as izip +import six + import actions from usertypes import get_type_default @@ -29,7 +31,7 @@ class TableDataSet(object): def apply_doc_action(self, action): try: getattr(self, action.__class__.__name__)(*action) - except Exception, e: + except Exception as e: log.warn("ERROR applying action %s: %s" % (action, e)) raise @@ -48,12 +50,12 @@ class TableDataSet(object): # Actions on records. #---------------------------------------- def AddRecord(self, table_id, row_id, columns): - self.BulkAddRecord(table_id, [row_id], {key: [val] for key, val in columns.iteritems()}) + self.BulkAddRecord(table_id, [row_id], {key: [val] for key, val in six.iteritems(columns)}) def BulkAddRecord(self, table_id, row_ids, columns): table_data = self.all_tables[table_id] table_data.row_ids.extend(row_ids) - for col, values in table_data.columns.iteritems(): + for col, values in six.iteritems(table_data.columns): if col in columns: values.extend(columns[col]) else: @@ -67,19 +69,19 @@ class TableDataSet(object): def BulkRemoveRecord(self, table_id, row_ids): table_data = self.all_tables[table_id] remove_set = set(row_ids) - for col, values in table_data.columns.iteritems(): + for col, values in six.iteritems(table_data.columns): values[:] = [v for r, v in izip(table_data.row_ids, values) if r not in remove_set] table_data.row_ids[:] = [r for r in table_data.row_ids if r not in remove_set] def UpdateRecord(self, table_id, row_id, columns): self.BulkUpdateRecord( - table_id, [row_id], {key: [val] for key, val in columns.iteritems()}) + table_id, [row_id], {key: [val] for key, val in six.iteritems(columns)}) def BulkUpdateRecord(self, table_id, row_ids, columns): table_data = self.all_tables[table_id] rowid_map = {r:i for i, r in enumerate(table_data.row_ids)} table_indices = [rowid_map[r] for r in row_ids] - for col, values in columns.iteritems(): + for col, values in six.iteritems(columns): if col in table_data.columns: col_values = table_data.columns[col] for i, v in izip(table_indices, values): @@ -88,7 +90,7 @@ class TableDataSet(object): def ReplaceTableData(self, table_id, row_ids, columns): table_data = self.all_tables[table_id] del table_data.row_ids[:] - for col, values in table_data.columns.iteritems(): + for col, values in six.iteritems(table_data.columns): del values[:] self.BulkAddRecord(table_id, row_ids, columns) diff --git a/sandbox/grist/test_acl_formula.py b/sandbox/grist/test_acl_formula.py index 3095975f..55826ea2 100644 --- a/sandbox/grist/test_acl_formula.py +++ b/sandbox/grist/test_acl_formula.py @@ -5,6 +5,7 @@ import unittest from acl_formula import parse_acl_formula import test_engine + class TestACLFormula(unittest.TestCase): def test_basic(self): # Test a few basic formulas and structures, hitting everything we expect to support @@ -104,19 +105,19 @@ class TestACLFormula(unittest.TestCase): self.assertRaises(SyntaxError, parse_acl_formula, "def foo(): pass") # Unsupported node type - self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "max(rec)") - self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "user.id in {1, 2, 3}") - self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "1 if user.IsAnon else 2") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "max(rec)") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "user.id in {1, 2, 3}") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 if user.IsAnon else 2") # Unsupported operation - self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "1 | 2") - self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "1 << 2") - self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "~test") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 | 2") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 << 2") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "~test") # Syntax error self.assertRaises(SyntaxError, parse_acl_formula, "[(]") self.assertRaises(SyntaxError, parse_acl_formula, "user.id in (1,2))") - self.assertRaisesRegexp(SyntaxError, r'invalid syntax on line 1 col 9', parse_acl_formula, "foo and !bar") + self.assertRaisesRegex(SyntaxError, r'invalid syntax on line 1 col 9', parse_acl_formula, "foo and !bar") class TestACLFormulaUserActions(test_engine.EngineTestCase): def test_acl_actions(self): diff --git a/sandbox/grist/test_codebuilder.py b/sandbox/grist/test_codebuilder.py index 9bdc7b91..2e19e740 100644 --- a/sandbox/grist/test_codebuilder.py +++ b/sandbox/grist/test_codebuilder.py @@ -2,6 +2,8 @@ import unittest import codebuilder +import six + def make_body(formula, default=None): return codebuilder.make_formula_body(formula, default).get_text() @@ -122,7 +124,7 @@ for a in rec: return rec """) - self.assertRegexpMatches(make_body(body), + self.assertRegex(make_body(body), r"raise SyntaxError\('Grist disallows assignment" + r" to the special variable \"rec\" on line 4 col 7'\)") @@ -135,8 +137,8 @@ return rec self.assertEqual(make_body(u"'résumé' + $foo"), u"return 'résumé' + rec.foo") # Check the return type of make_body() - self.assertEqual(type(make_body("foo")), unicode) - self.assertEqual(type(make_body(u"foo")), unicode) + self.assertEqual(type(make_body("foo")), six.text_type) + self.assertEqual(type(make_body(u"foo")), six.text_type) def test_wrap_logical(self): diff --git a/sandbox/grist/test_column_actions.py b/sandbox/grist/test_column_actions.py index 1adfaa2e..ea9d86be 100644 --- a/sandbox/grist/test_column_actions.py +++ b/sandbox/grist/test_column_actions.py @@ -338,7 +338,7 @@ class TestColumnActions(test_engine.EngineTestCase): self.init_sample_data() # Test that we cannot remove group-by columns from summary tables directly. - with self.assertRaisesRegexp(ValueError, "cannot remove .* group-by"): + with self.assertRaisesRegex(ValueError, "cannot remove .* group-by"): self.apply_user_action(["BulkRemoveRecord", '_grist_Tables_column', [20,18]]) # Test that group-by columns in summary tables get removed. diff --git a/sandbox/grist/test_docmodel.py b/sandbox/grist/test_docmodel.py index b4558f84..1f3d1a53 100644 --- a/sandbox/grist/test_docmodel.py +++ b/sandbox/grist/test_docmodel.py @@ -244,6 +244,6 @@ class TestDocModel(test_engine.EngineTestCase): # Verify that positions are set such that the order is what we asked for. student_columns = self.engine.docmodel.tables.lookupOne(tableId='Students').columns - self.assertEqual(map(int, student_columns), [1,2,4,5,6,25,22,23]) + self.assertEqual(list(map(int, student_columns)), [1,2,4,5,6,25,22,23]) school_columns = self.engine.docmodel.tables.lookupOne(tableId='Schools').columns - self.assertEqual(map(int, school_columns), [24,10,12]) + self.assertEqual(list(map(int, school_columns)), [24,10,12]) diff --git a/sandbox/grist/test_engine.py b/sandbox/grist/test_engine.py index 5964a8c4..75fbe33b 100644 --- a/sandbox/grist/test_engine.py +++ b/sandbox/grist/test_engine.py @@ -4,6 +4,8 @@ import json import unittest from collections import namedtuple +import six + import actions import column import engine @@ -22,6 +24,8 @@ View = namedtuple('View', 'id sections') Section = namedtuple('Section', 'id parentKey tableRef fields') Field = namedtuple('Field', 'id colRef') +unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp +unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches class EngineTestCase(unittest.TestCase): """ @@ -152,9 +156,9 @@ class EngineTestCase(unittest.TestCase): # Convert observed and expected actions into a comparable form. for k in self.action_group_action_fields: if k in observed: - observed[k] = map(get_comparable_repr, observed[k]) + observed[k] = [get_comparable_repr(v) for v in observed[k]] if k in expected: - expected[k] = map(get_comparable_repr, expected[k]) + expected[k] = [get_comparable_repr(v) for v in expected[k]] if observed != expected: o_lines = self._formatActionGroup(observed) @@ -192,13 +196,13 @@ class EngineTestCase(unittest.TestCase): output = {t: self.engine.fetch_table(t) for t in self.engine.schema} output = testutil.replace_nans(output) output = actions.encode_objects(output) - print ''.join(self._getEngineDataLines(output)) + print(''.join(self._getEngineDataLines(output))) def dump_actions(self, out_actions): """ Prints out_actions in human-readable format, for help in writing / debugging tets. """ - print "\n".join(self._formatActionGroup(out_actions.__dict__)) + print("\n".join(self._formatActionGroup(out_actions.__dict__))) def assertTableData(self, table_name, data=[], cols="all", rows="all", sort=None): """ @@ -237,7 +241,7 @@ class EngineTestCase(unittest.TestCase): if sort: row_ids.sort(key=lambda r: sort(table.get_record(r))) - observed_col_data = {c.col_id: map(c.raw_get, row_ids) for c in columns if c.col_id != "id"} + observed_col_data = {c.col_id: [c.raw_get(r) for r in row_ids] for c in columns if c.col_id != "id"} observed = actions.TableData(table_name, row_ids, observed_col_data) self.assertEqualDocData({table_name: observed}, {table_name: expected}, col_names=col_names) @@ -286,7 +290,7 @@ class EngineTestCase(unittest.TestCase): """ schema = sample["SCHEMA"] self.engine.load_meta_tables(schema['_grist_Tables'], schema['_grist_Tables_column']) - for data in sample["DATA"].itervalues(): + for data in six.itervalues(sample["DATA"]): self.engine.load_table(data) self.engine.load_done() @@ -426,11 +430,11 @@ class TestEngine(EngineTestCase): sample = self.samples[data.pop("USE_SAMPLE")] expected_data = sample["DATA"].copy() expected_data.update({t: testutil.table_data_from_rows(t, tdata[0], tdata[1:]) - for (t, tdata) in data.iteritems()}) + for (t, tdata) in six.iteritems(data)}) self.assertCorrectEngineData(expected_data) else: raise ValueError("Unrecognized step %s in test script" % step) - except Exception, e: + except Exception as e: prefix = "LINE %s: " % line e.args = (prefix + e.args[0],) + e.args[1:] if e.args else (prefix,) raise @@ -526,7 +530,7 @@ class TestEngine(EngineTestCase): # Simulate an error inside a DocAction, and make sure we restore the schema (don't leave it in # inconsistent with metadata). self.load_sample(testutil.parse_test_sample(self.sample1)) - with self.assertRaisesRegexp(AttributeError, r"'BAD'"): + with self.assertRaisesRegex(AttributeError, r"'BAD'"): self.add_column('Address', 'bad', isFormula=False, type="BAD") self.engine.assert_schema_consistent() diff --git a/sandbox/grist/test_formula_error.py b/sandbox/grist/test_formula_error.py index 9bda388a..e2605c28 100644 --- a/sandbox/grist/test_formula_error.py +++ b/sandbox/grist/test_formula_error.py @@ -2,6 +2,9 @@ Tests that formula error messages (traceback) are correct """ import textwrap + +import six + import depend import test_engine import testutil @@ -48,14 +51,19 @@ else: self.assertIsInstance(exc.error, type_) self.assertEqual(str(exc.error), message) if tracebackRegexp: - self.assertRegexpMatches(exc.details, tracebackRegexp) + self.assertRegex(exc.details, tracebackRegexp) def test_formula_errors(self): self.load_sample(self.sample) - self.assertFormulaError(self.engine.get_formula_error('Math', 'excel_formula', 3), - TypeError, 'SQRT() takes exactly 1 argument (2 given)', - r"TypeError: SQRT\(\) takes exactly 1 argument \(2 given\)") + if six.PY2: + self.assertFormulaError(self.engine.get_formula_error('Math', 'excel_formula', 3), + TypeError, 'SQRT() takes exactly 1 argument (2 given)', + r"TypeError: SQRT\(\) takes exactly 1 argument \(2 given\)") + else: + self.assertFormulaError(self.engine.get_formula_error('Math', 'excel_formula', 3), + TypeError, 'SQRT() takes 1 positional argument but 2 were given', + r"TypeError: SQRT\(\) takes 1 positional argument but 2 were given") self.assertFormulaError(self.engine.get_formula_error('Math', 'built_in_formula', 3), TypeError, "'int' object is not iterable") diff --git a/sandbox/grist/test_gencode.py b/sandbox/grist/test_gencode.py index 0bd8d4a7..07fa9894 100644 --- a/sandbox/grist/test_gencode.py +++ b/sandbox/grist/test_gencode.py @@ -2,6 +2,8 @@ import unittest import difflib import re +from six.moves import xrange + import gencode import identifiers import records diff --git a/sandbox/grist/test_gpath.py b/sandbox/grist/test_gpath.py index d068d96c..ca442f81 100644 --- a/sandbox/grist/test_gpath.py +++ b/sandbox/grist/test_gpath.py @@ -36,10 +36,10 @@ class TestGpath(unittest.TestCase): self.assertEqual(self.obj["hello"], "blah") def test_set_strict(self): - with self.assertRaisesRegexp(Exception, r"non-existent"): + with self.assertRaisesRegex(Exception, r"non-existent"): gpath.place(self.obj, ["bar", 4], 17) - with self.assertRaisesRegexp(Exception, r"not a plain object"): + with self.assertRaisesRegex(Exception, r"not a plain object"): gpath.place(self.obj, ["foo", 0], 17) @@ -54,13 +54,13 @@ class TestGpath(unittest.TestCase): ["asdf", {"bar": 1}, {"bar": 2}, "hello", {"baz": 3}, "world"]) def test_insert_strict(self): - with self.assertRaisesRegexp(Exception, r'not an array'): + with self.assertRaisesRegex(Exception, r'not an array'): gpath.insert(self.obj, ["foo"], "asdf") - with self.assertRaisesRegexp(Exception, r'invalid.*index'): + with self.assertRaisesRegex(Exception, r'invalid.*index'): gpath.insert(self.obj, ["foo", -1], 17) - with self.assertRaisesRegexp(Exception, r'invalid.*index'): + with self.assertRaisesRegex(Exception, r'invalid.*index'): gpath.insert(self.obj, ["foo", "foo"], 17) def test_update(self): @@ -75,13 +75,13 @@ class TestGpath(unittest.TestCase): def test_update_strict(self): """update should be strict""" - with self.assertRaisesRegexp(Exception, r'non-existent'): + with self.assertRaisesRegex(Exception, r'non-existent'): gpath.update(self.obj, ["bar", 4], 17) - with self.assertRaisesRegexp(Exception, r'not an array'): + with self.assertRaisesRegex(Exception, r'not an array'): gpath.update(self.obj, ["foo"], 17) - with self.assertRaisesRegexp(Exception, r'invalid.*index'): + with self.assertRaisesRegex(Exception, r'invalid.*index'): gpath.update(self.obj, ["foo", -1], 17) - with self.assertRaisesRegexp(Exception, r'invalid.*index'): + with self.assertRaisesRegex(Exception, r'invalid.*index'): gpath.update(self.obj, ["foo", None], 17) def test_remove(self): @@ -96,13 +96,13 @@ class TestGpath(unittest.TestCase): def test_remove_strict(self): """remove should be strict""" - with self.assertRaisesRegexp(Exception, r'non-existent'): + with self.assertRaisesRegex(Exception, r'non-existent'): gpath.remove(self.obj, ["bar", 4]) - with self.assertRaisesRegexp(Exception, r'not an array'): + with self.assertRaisesRegex(Exception, r'not an array'): gpath.remove(self.obj, ["foo"]) - with self.assertRaisesRegexp(Exception, r'invalid.*index'): + with self.assertRaisesRegex(Exception, r'invalid.*index'): gpath.remove(self.obj, ["foo", -1]) - with self.assertRaisesRegexp(Exception, r'invalid.*index'): + with self.assertRaisesRegex(Exception, r'invalid.*index'): gpath.remove(self.obj, ["foo", None]) def test_glob(self): @@ -112,7 +112,7 @@ class TestGpath(unittest.TestCase): self.assertEqual(gpath.place(self.obj, ["foo", "*", "bar"], 17), 3) self.assertEqual(self.obj["foo"], [{"bar": 17}, {"bar": 17}, {"baz": 3, "bar": 17}]) - with self.assertRaisesRegexp(Exception, r'non-existent object at \/foo\/\*\/bad'): + with self.assertRaisesRegex(Exception, r'non-existent object at \/foo\/\*\/bad'): gpath.place(self.obj, ["foo", "*", "bad", "test"], 10) self.assertEqual(gpath.update(self.obj, ["foo", "*"], "hello"), 3) @@ -120,9 +120,9 @@ class TestGpath(unittest.TestCase): def test_glob_strict_wildcard(self): """should only support tail wildcard for updates""" - with self.assertRaisesRegexp(Exception, r'invalid array index'): + with self.assertRaisesRegex(Exception, r'invalid array index'): gpath.remove(self.obj, ["foo", "*"]) - with self.assertRaisesRegexp(Exception, r'invalid array index'): + with self.assertRaisesRegex(Exception, r'invalid array index'): gpath.insert(self.obj, ["foo", "*"], 1) def test_glob_wildcard_keys(self): @@ -132,7 +132,7 @@ class TestGpath(unittest.TestCase): self.assertEqual(gpath.place(self.obj, ["foo", 0, "*"], 17), 1) self.assertEqual(self.obj["foo"], [{"bar": 1, '*': 17}, {"bar": 2}, {"baz": 3}]) - with self.assertRaisesRegexp(Exception, r'non-existent'): + with self.assertRaisesRegex(Exception, r'non-existent'): gpath.place(self.obj, ["*", 0, "bar"], 17) def test_glob_nested(self): diff --git a/sandbox/grist/test_import_actions.py b/sandbox/grist/test_import_actions.py index 1514467d..9766e72f 100644 --- a/sandbox/grist/test_import_actions.py +++ b/sandbox/grist/test_import_actions.py @@ -13,7 +13,7 @@ class TestImportActions(test_engine.EngineTestCase): {'id': 'Zip', 'type': 'Int'}]]) self.apply_user_action(['BulkAddRecord', 'Source', [1, 2], {'Name': ['John', 'Alison'], 'City': ['New York', 'Boston'], - 'Zip': [03011, 07003]}]) + 'Zip': [3011, 7003]}]) self.assertTableData('_grist_Tables_column', cols="subset", data=[ ["id", "colId", "type", "isFormula", "formula"], [1, "manualSort", "ManualSortPos", False, ""], @@ -79,8 +79,8 @@ class TestImportActions(test_engine.EngineTestCase): self.assertTableData('Source', cols="all", data=[ ["id", "Name", "City", "Zip", "gristHelper_Import_Name", "gristHelper_Import_City", "manualSort"], - [1, "John", "New York", 03011, "John", "New York", 1.0], - [2, "Alison", "Boston", 07003, "Alison", "Boston", 2.0], + [1, "John", "New York", 3011, "John", "New York", 1.0], + [2, "Alison", "Boston", 7003, "Alison", "Boston", 2.0], ]) self.assertPartialData("_grist_Views_section", ["id", "tableRef", 'fields'], [ @@ -107,8 +107,8 @@ class TestImportActions(test_engine.EngineTestCase): self.assertTableData('Source', cols="all", data=[ ["id", "Name", "City", "Zip", "gristHelper_Import_State", "manualSort"], - [1, "John", "New York", 03011, "", 1.0], - [2, "Alison", "Boston", 07003, "", 2.0], + [1, "John", "New York", 3011, "", 1.0], + [2, "Alison", "Boston", 7003, "", 2.0], ]) self.assertPartialData("_grist_Views_section", ["id", "tableRef", 'fields'], [ [1, 1, [1, 2, 3]], @@ -168,8 +168,8 @@ class TestImportActions(test_engine.EngineTestCase): self.assertTableData('Source', cols="all", data=[ ["id", "Name", "City", "Zip", "gristHelper_Import_Name", "gristHelper_Import_City", "gristHelper_Import_Zip", "manualSort"], - [1, "John", "New York", 03011, "John", "New York", 03011, 1.0], - [2, "Alison", "Boston", 07003, "Alison", "Boston", 07003, 2.0], + [1, "John", "New York", 3011, "John", "New York", 3011, 1.0], + [2, "Alison", "Boston", 7003, "Alison", "Boston", 7003, 2.0], ]) self.assertPartialData("_grist_Views_section", ["id", "tableRef", 'fields'], [ [1, 1, [1, 2, 3]], diff --git a/sandbox/grist/test_match_counter.py b/sandbox/grist/test_match_counter.py index 7d135dff..b84ca7b0 100644 --- a/sandbox/grist/test_match_counter.py +++ b/sandbox/grist/test_match_counter.py @@ -3,6 +3,10 @@ import string import timeit import unittest from collections import Hashable + +import six +from six.moves import xrange + import match_counter from testutil import repeat_until_passes @@ -22,7 +26,7 @@ class MatchCounterOther(object): pass matches = 0 - for v, n in self.sample_counts.iteritems(): + for v, n in six.iteritems(self.sample_counts): if n > 0: matches += 1 self.sample_counts[v] = 0 diff --git a/sandbox/grist/test_migrations.py b/sandbox/grist/test_migrations.py index ff24aa54..9618175a 100644 --- a/sandbox/grist/test_migrations.py +++ b/sandbox/grist/test_migrations.py @@ -1,5 +1,7 @@ import unittest +import six + import actions import schema import table_data_set @@ -23,7 +25,7 @@ class TestMigrations(unittest.TestCase): # Figure out the missing actions. doc_actions = [] - for table_id in sorted(current_schema.viewkeys() | migrated_schema.viewkeys()): + for table_id in sorted(six.viewkeys(current_schema) | six.viewkeys(migrated_schema)): if table_id not in migrated_schema: doc_actions.append(actions.AddTable(table_id, current_schema[table_id].values())) elif table_id not in current_schema: @@ -31,7 +33,7 @@ class TestMigrations(unittest.TestCase): else: current_cols = current_schema[table_id] migrated_cols = migrated_schema[table_id] - for col_id in sorted(current_cols.viewkeys() | migrated_cols.viewkeys()): + for col_id in sorted(six.viewkeys(current_cols) | six.viewkeys(migrated_cols)): if col_id not in migrated_cols: doc_actions.append(actions.AddColumn(table_id, col_id, current_cols[col_id])) elif col_id not in current_cols: @@ -39,7 +41,7 @@ class TestMigrations(unittest.TestCase): else: current_info = current_cols[col_id] migrated_info = migrated_cols[col_id] - delta = {k: v for k, v in current_info.iteritems() if v != migrated_info.get(k)} + delta = {k: v for k, v in six.iteritems(current_info) if v != migrated_info.get(k)} if delta: doc_actions.append(actions.ModifyColumn(table_id, col_id, delta)) diff --git a/sandbox/grist/test_moment.py b/sandbox/grist/test_moment.py index ddb63afb..c5f21a09 100644 --- a/sandbox/grist/test_moment.py +++ b/sandbox/grist/test_moment.py @@ -21,7 +21,7 @@ class TestMoment(unittest.TestCase): [datetime(1979, 10, 28, 6, 0, 0), 309938400000, "EST", 300, 1, 0], # - 2037 - [datetime(2037, 3, 8, 6, 59, 59), 2120108399000, "EST", 300, 1, 59], - [datetime(2037, 03, 8, 7, 0, 0), 2120108400000, "EDT", 240, 3, 0], + [datetime(2037, 3, 8, 7, 0, 0), 2120108400000, "EDT", 240, 3, 0], [datetime(2037, 11, 1, 5, 59, 59), 2140667999000, "EDT", 240, 1, 59] ] new_york_errors = [ @@ -255,7 +255,7 @@ class TestMoment(unittest.TestCase): def test_dt_to_ds(self): # Verify that dt_to_ts works for both naive and aware datetime objects. - value_dt = datetime(2015, 03, 14, 0, 0) # In UTC + value_dt = datetime(2015, 3, 14, 0, 0) # In UTC value_sec = 1426291200 tzla = moment.get_zone('America/Los_Angeles') def format_utc(ts): @@ -287,7 +287,7 @@ class TestMoment(unittest.TestCase): self.assertEqual(value_dt_aware.strftime(fmt), '2015-02-13 20:00:00 EST') def test_date_to_ts(self): - d = date(2015, 03, 14) + d = date(2015, 3, 14) tzla = moment.get_zone('America/Los_Angeles') def format_utc(ts): return moment.ts_to_dt(ts, moment.get_zone('UTC')).strftime(fmt) diff --git a/sandbox/grist/test_record_func.py b/sandbox/grist/test_record_func.py index 8a1a1de5..520b2046 100644 --- a/sandbox/grist/test_record_func.py +++ b/sandbox/grist/test_record_func.py @@ -7,6 +7,7 @@ import objtypes import testsamples import testutil import test_engine +from objtypes import RecordStub log = logger.Logger(__name__, logger.INFO) @@ -18,84 +19,84 @@ class TestRecordFunc(test_engine.EngineTestCase): def test_record_self(self): self.load_sample(testsamples.sample_students) - self.add_column("Schools", "Foo", formula='repr(RECORD(rec))') + self.add_column("Schools", "Foo", formula='RECORD(rec)') self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"], - [2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"], - [3, "{'address': Address[13], 'id': 3, 'name': 'Yale'}"], - [4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"], + [1, {'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}], + [2, {'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}], + [3, {'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}], + [4, {'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}], ]) # A change to data is reflected self.update_record("Schools", 3, name="UConn") self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"], - [2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"], - [3, "{'address': Address[13], 'id': 3, 'name': 'UConn'}"], - [4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"], + [1, {'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}], + [2, {'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}], + [3, {'address': RecordStub('Address', 13), 'id': 3, 'name': 'UConn'}], + [4, {'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}], ]) # A column addition is reflected self.add_column("Schools", "Bar", formula='len($name)') self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'address': Address[11], 'Bar': 8, 'id': 1, 'name': 'Columbia'}"], - [2, "{'address': Address[12], 'Bar': 8, 'id': 2, 'name': 'Columbia'}"], - [3, "{'address': Address[13], 'Bar': 5, 'id': 3, 'name': 'UConn'}"], - [4, "{'address': Address[14], 'Bar': 4, 'id': 4, 'name': 'Yale'}"], + [1, {'address': RecordStub('Address', 11), 'Bar': 8, 'id': 1, 'name': 'Columbia'}], + [2, {'address': RecordStub('Address', 12), 'Bar': 8, 'id': 2, 'name': 'Columbia'}], + [3, {'address': RecordStub('Address', 13), 'Bar': 5, 'id': 3, 'name': 'UConn'}], + [4, {'address': RecordStub('Address', 14), 'Bar': 4, 'id': 4, 'name': 'Yale'}], ]) def test_reference(self): self.load_sample(testsamples.sample_students) - self.add_column("Schools", "Foo", formula='repr(RECORD($address))') + self.add_column("Schools", "Foo", formula='RECORD($address)') self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'city': 'New York', 'id': 11}"], - [2, "{'city': 'Colombia', 'id': 12}"], - [3, "{'city': 'New Haven', 'id': 13}"], - [4, "{'city': 'West Haven', 'id': 14}"], + [1, {'city': 'New York', 'id': 11}], + [2, {'city': 'Colombia', 'id': 12}], + [3, {'city': 'New Haven', 'id': 13}], + [4, {'city': 'West Haven', 'id': 14}], ]) # A change to referenced data is still reflected; try a different kind of change here self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"]) self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'ciudad': 'New York', 'id': 11}"], - [2, "{'ciudad': 'Colombia', 'id': 12}"], - [3, "{'ciudad': 'New Haven', 'id': 13}"], - [4, "{'ciudad': 'West Haven', 'id': 14}"], + [1, {'ciudad': 'New York', 'id': 11}], + [2, {'ciudad': 'Colombia', 'id': 12}], + [3, {'ciudad': 'New Haven', 'id': 13}], + [4, {'ciudad': 'West Haven', 'id': 14}], ]) def test_record_expand_refs(self): self.load_sample(testsamples.sample_students) - self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))') + self.add_column("Schools", "Foo", formula='RECORD(rec, expand_refs=1)') self.add_column("Address", "student", type="Ref:Students") self.update_record("Address", 12, student=6) self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'address': {'city': 'New York', 'id': 11, 'student': Students[0]}," + - " 'id': 1, 'name': 'Columbia'}"], - [2, "{'address': {'city': 'Colombia', 'id': 12, 'student': Students[6]}," + - " 'id': 2, 'name': 'Columbia'}"], - [3, "{'address': {'city': 'New Haven', 'id': 13, 'student': Students[0]}," + - " 'id': 3, 'name': 'Yale'}"], - [4, "{'address': {'city': 'West Haven', 'id': 14, 'student': Students[0]}," + - " 'id': 4, 'name': 'Yale'}"], + [1, {'address': {'city': 'New York', 'id': 11, 'student': RecordStub("Students", 0)}, + 'id': 1, 'name': 'Columbia'}], + [2, {'address': {'city': 'Colombia', 'id': 12, 'student': RecordStub("Students", 6)}, + 'id': 2, 'name': 'Columbia'}], + [3, {'address': {'city': 'New Haven', 'id': 13, 'student': RecordStub("Students", 0)}, + 'id': 3, 'name': 'Yale'}], + [4, {'address': {'city': 'West Haven', 'id': 14, 'student': RecordStub("Students", 0)}, + 'id': 4, 'name': 'Yale'}], ]) - self.modify_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=2))') + self.modify_column("Schools", "Foo", formula='RECORD(rec, expand_refs=2)') self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'address': {'city': 'New York', 'id': 11, 'student': None}," + - " 'id': 1, 'name': 'Columbia'}"], - [2, "{'address': {'city': 'Colombia', 'id': 12, " + - "'student': {'firstName': 'Gerald', 'schoolName': 'Yale', 'lastName': 'Ford', " + - "'schoolCities': 'New Haven:West Haven', 'schoolIds': '3:4', 'id': 6}}," + - " 'id': 2, 'name': 'Columbia'}"], - [3, "{'address': {'city': 'New Haven', 'id': 13, 'student': None}," + - " 'id': 3, 'name': 'Yale'}"], - [4, "{'address': {'city': 'West Haven', 'id': 14, 'student': None}," + - " 'id': 4, 'name': 'Yale'}"], + [1, {'address': {'city': 'New York', 'id': 11, 'student': None}, + 'id': 1, 'name': 'Columbia'}], + [2, {'address': {'city': 'Colombia', 'id': 12, + 'student': {'firstName': 'Gerald', 'schoolName': 'Yale', 'lastName': 'Ford', + 'schoolCities': 'New Haven:West Haven', 'schoolIds': '3:4', 'id': 6}}, + 'id': 2, 'name': 'Columbia'}], + [3, {'address': {'city': 'New Haven', 'id': 13, 'student': None}, + 'id': 3, 'name': 'Yale'}], + [4, {'address': {'city': 'West Haven', 'id': 14, 'student': None}, + 'id': 4, 'name': 'Yale'}], ]) def test_record_date_options(self): self.load_sample(testsamples.sample_students) - self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))') + self.add_column("Schools", "Foo", formula='RECORD(rec, expand_refs=1)') self.add_column("Address", "DT", type='DateTime') self.add_column("Address", "D", type='Date', formula="$DT and $DT.date()") self.update_records("Address", ['id', 'DT'], [ @@ -106,61 +107,57 @@ class TestRecordFunc(test_engine.EngineTestCase): d1 = datetime.datetime(2020, 9, 13, 8, 26, 40, tzinfo=moment.tzinfo('America/New_York')) d2 = datetime.datetime(2017, 7, 13, 22, 40, tzinfo=moment.tzinfo('America/New_York')) self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'address': {'city': 'New York', 'DT': %s, 'id': 11, 'D': %s}, " % - (repr(d1), repr(d1.date())) + - "'id': 1, 'name': 'Columbia'}"], - [2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " + - "'id': 2, 'name': 'Columbia'}"], - [3, "{'address': {'city': 'New Haven', 'DT': %s, 'id': 13, 'D': %s}, " % - (repr(d2), repr(d2.date())) + - "'id': 3, 'name': 'Yale'}"], - [4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " + - "'id': 4, 'name': 'Yale'}"], + [1, {'address': {'city': 'New York', 'DT': d1, 'id': 11, 'D': d1.date()}, + 'id': 1, 'name': 'Columbia'}], + [2, {'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, + 'id': 2, 'name': 'Columbia'}], + [3, {'address': {'city': 'New Haven', 'DT': d2, 'id': 13, 'D': d2.date()}, + 'id': 3, 'name': 'Yale'}], + [4, {'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, + 'id': 4, 'name': 'Yale'}], ]) self.modify_column("Schools", "Foo", - formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=True))') + formula='RECORD(rec, expand_refs=1, dates_as_iso=True)') self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " % - (d1.isoformat(), d1.date().isoformat()) + - "'id': 1, 'name': 'Columbia'}"], - [2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " + - "'id': 2, 'name': 'Columbia'}"], - [3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " % - (d2.isoformat(), d2.date().isoformat()) + - "'id': 3, 'name': 'Yale'}"], - [4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " + - "'id': 4, 'name': 'Yale'}"], + [1, {'address': {'city': 'New York', 'DT': d1.isoformat(), 'id': 11, 'D': d1.date().isoformat()}, + 'id': 1, 'name': 'Columbia'}], + [2, {'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, + 'id': 2, 'name': 'Columbia'}], + [3, {'address': {'city': 'New Haven', 'DT': d2.isoformat(), 'id': 13, 'D': d2.date().isoformat()}, + 'id': 3, 'name': 'Yale'}], + [4, {'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, + 'id': 4, 'name': 'Yale'}], ]) def test_record_set(self): self.load_sample(testsamples.sample_students) self.add_column("Students", "schools", formula='Schools.lookupRecords(name=$schoolName)') - self.add_column("Students", "Foo", formula='repr(RECORD($schools))') + self.add_column("Students", "Foo", formula='RECORD($schools)') self.assertPartialData("Students", ["id", "Foo"], [ - [1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," + - " {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"], - [2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," + - " {'address': Address[14], 'id': 4, 'name': 'Yale'}]"], - [3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," + - " {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"], - [4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," + - " {'address': Address[14], 'id': 4, 'name': 'Yale'}]"], - [5, "[]"], - [6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," + - " {'address': Address[14], 'id': 4, 'name': 'Yale'}]"], + [1, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}, + {'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}]], + [2, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}, + {'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}]], + [3, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}, + {'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}]], + [4, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}, + {'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}]], + [5, []], + [6, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}, + {'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}]], ]) # Try a field with filtered lookupRecords result, as an iterable. self.modify_column("Students", "Foo", - formula='repr(RECORD(s for s in $schools if s.address.city.startswith("New")))') + formula='RECORD(s for s in $schools if s.address.city.startswith("New"))') self.assertPartialData("Students", ["id", "Foo"], [ - [1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"], - [2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"], - [3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"], - [4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"], - [5, "[]"], - [6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"], + [1, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}]], + [2, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}]], + [3, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}]], + [4, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}]], + [5, []], + [6, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}]], ]) def test_record_bad_calls(self): @@ -172,25 +169,25 @@ class TestRecordFunc(test_engine.EngineTestCase): [3, objtypes.RaisedException(ValueError())], [4, objtypes.RaisedException(ValueError())], ]) - self.modify_column("Schools", "Foo", formula='repr(RECORD([rec] if $id == 2 else $id))') + self.modify_column("Schools", "Foo", formula='repr(sorted(RECORD(rec if $id == 2 else $id).items()))') self.assertPartialData("Schools", ["id", "Foo"], [ [1, objtypes.RaisedException(ValueError())], - [2, "[{'address': Address[12], 'id': 2, 'name': 'Columbia'}]"], + [2, "[('address', Address[12]), ('id', 2), ('name', 'Columbia')]"], [3, objtypes.RaisedException(ValueError())], [4, objtypes.RaisedException(ValueError())], ]) - self.assertEqual(self.engine.get_formula_error('Schools', 'Foo', 1).error.message, + self.assertEqual(str(self.engine.get_formula_error('Schools', 'Foo', 1).error), 'RECORD() requires a Record or an iterable of Records') def test_record_error_cells(self): self.load_sample(testsamples.sample_students) - self.add_column("Schools", "Foo", formula='repr(RECORD($address))') - self.add_column("Address", "Bar", formula='$id/($id%2)') + self.add_column("Schools", "Foo", formula='RECORD($address)') + self.add_column("Address", "Bar", formula='$id//($id%2)') self.assertPartialData("Schools", ["id", "Foo"], [ - [1, "{'city': 'New York', 'Bar': 11, 'id': 11}"], - [2, "{'city': 'Colombia', 'Bar': None, 'id': 12, " + - "'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"], - [3, "{'city': 'New Haven', 'Bar': 13, 'id': 13}"], - [4, "{'city': 'West Haven', 'Bar': None, 'id': 14, " + - "'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"], + [1, {'city': 'New York', 'Bar': 11, 'id': 11}], + [2, {'city': 'Colombia', 'Bar': None, 'id': 12, + '_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}], + [3, {'city': 'New Haven', 'Bar': 13, 'id': 13}], + [4, {'city': 'West Haven', 'Bar': None, 'id': 14, + '_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}], ]) diff --git a/sandbox/grist/test_relabeling.py b/sandbox/grist/test_relabeling.py index 7cd5ac83..6ca6043f 100644 --- a/sandbox/grist/test_relabeling.py +++ b/sandbox/grist/test_relabeling.py @@ -1,7 +1,8 @@ import relabeling from sortedcontainers import SortedListWithKey -from itertools import izip +from six.moves import zip as izip, xrange + import unittest import sys @@ -264,7 +265,7 @@ class TestRelabeling(unittest.TestCase): slist.insert_items([(i, float('-inf')), (-i, float('inf'))]) self.assertEqual(slist.get_values(), - rev_range(2000) + [v for v,k in initial] + range(0, -2000, -1)) + rev_range(2000) + [v for v,k in initial] + list(xrange(0, -2000, -1))) #print slist.num_update_events, slist.num_updated_keys self.assertLess(slist.avg_updated_keys(), 3) self.assertLess(slist.num_update_events, 80) @@ -276,7 +277,7 @@ class TestRelabeling(unittest.TestCase): slist.insert_items([(i, ins_item.key)]) # Check the end result - self.assertEqual(slist.get_values(), ['a', 'b'] + range(1000) + ['c', 'd']) + self.assertEqual(slist.get_values(), ['a', 'b'] + list(xrange(1000)) + ['c', 'd']) self.assertAlmostEqual(slist.avg_updated_keys(), 3.5, delta=1) self.assertLess(slist.num_update_events, 40) @@ -299,7 +300,7 @@ class TestRelabeling(unittest.TestCase): ins_item = slist.find_value('c') for i in xrange(1000): slist.insert_items([(i, ins_item.key)], prepare_inserts=r.prepare_inserts_dumb) - self.assertEqual(slist.get_values(), ['a', 'b'] + range(1000) + ['c', 'd']) + self.assertEqual(slist.get_values(), ['a', 'b'] + list(xrange(1000)) + ['c', 'd']) self.assertGreater(slist.avg_updated_keys(), 8) def test_renumber_right_dumb(self): @@ -334,7 +335,7 @@ class TestRelabeling(unittest.TestCase): # aL1, al0, al1, a, ar1, ar0, aR1, ... # aL1, al0, aL2, al1, al2, a, ar2, ar1, aR2, ar0, aR1, ... def left_half(val): - half = range(2*N - 1) + half = list(xrange(2*N - 1)) half[0::2] = ['%sL%d' % (val, i) for i in xrange(1, N + 1)] half[1::2] = ['%sl%d' % (val, i) for i in xrange(0, N - 1)] half[-1] = '%sl%d' % (val, N - 1) @@ -355,7 +356,7 @@ class TestRelabeling(unittest.TestCase): def rev_range(n): - return list(reversed(range(n))) + return list(reversed(list(xrange(n)))) if __name__ == "__main__": unittest.main() diff --git a/sandbox/grist/test_summary2.py b/sandbox/grist/test_summary2.py index 15442241..adca7a1d 100644 --- a/sandbox/grist/test_summary2.py +++ b/sandbox/grist/test_summary2.py @@ -31,7 +31,7 @@ class TestSummary2(test_engine.EngineTestCase): self.apply_user_action(["CreateViewSection", 1, 0, "record", []]) # Check that we cannot add a non-formula column. - with self.assertRaisesRegexp(ValueError, r'non-formula column'): + with self.assertRaisesRegex(ValueError, r'non-formula column'): self.apply_user_action(["AddColumn", "GristSummary_7_Address", "average", {"type": "Text", "isFormula": False}]) @@ -171,7 +171,7 @@ class TestSummary2(test_engine.EngineTestCase): # Check that we cannot rename a summary group-by column. (Perhaps it's better to raise an # exception, but currently we translate the invalid request to a no-op.) - with self.assertRaisesRegexp(ValueError, r'Cannot modify .* group-by'): + with self.assertRaisesRegex(ValueError, r'Cannot modify .* group-by'): self.apply_user_action(["RenameColumn", "GristSummary_7_Address", "state", "s"]) # Verify all data. We'll repeat this after renamings to make sure there are no errors. @@ -353,46 +353,46 @@ class TestSummary2(test_engine.EngineTestCase): ]) # (1) no adding/removing/renaming non-formula columns. - with self.assertRaisesRegexp(ValueError, r'non-formula column'): + with self.assertRaisesRegex(ValueError, r'non-formula column'): self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo", {"type": "Numeric", "isFormula": False}]) - with self.assertRaisesRegexp(ValueError, r'group-by column'): + with self.assertRaisesRegex(ValueError, r'group-by column'): self.apply_user_action(["RemoveColumn", "GristSummary_7_Address", "state"]) - with self.assertRaisesRegexp(ValueError, r'Cannot modify .* group-by'): + with self.assertRaisesRegex(ValueError, r'Cannot modify .* group-by'): self.apply_user_action(["RenameColumn", "GristSummary_7_Address", "state", "st"]) # (2) no converting between formula/non-formula - with self.assertRaisesRegexp(ValueError, r'Cannot change .* formula and data'): + with self.assertRaisesRegex(ValueError, r'Cannot change .* formula and data'): self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "amount", {"isFormula": False}]) - with self.assertRaisesRegexp(ValueError, r'Cannot change .* formula and data'): + with self.assertRaisesRegex(ValueError, r'Cannot change .* formula and data'): self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "state", {"isFormula": True}]) # (3) no editing values in non-formula columns - with self.assertRaisesRegexp(ValueError, r'Cannot enter data .* group-by'): + with self.assertRaisesRegex(ValueError, r'Cannot enter data .* group-by'): self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 6, {"state": "ny"}]) # (4) no removing rows (this is questionable b/c empty rows might be OK to remove) - with self.assertRaisesRegexp(ValueError, r'Cannot remove record .* summary'): + with self.assertRaisesRegex(ValueError, r'Cannot remove record .* summary'): self.apply_user_action(["RemoveRecord", "GristSummary_7_Address", 6]) # (5) no renaming summary tables. - with self.assertRaisesRegexp(ValueError, r'cannot rename .* summary'): + with self.assertRaisesRegex(ValueError, r'cannot rename .* summary'): self.apply_user_action(["RenameTable", "GristSummary_7_Address", "GristSummary_hello"]) # Check that we can add an empty column, then set a formula for it. self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo", {}]) self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "foo", {"formula": "1+1"}]) - with self.assertRaisesRegexp(ValueError, "Can't save .* to formula"): + with self.assertRaisesRegex(ValueError, "Can't save .* to formula"): self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 1, {"foo": "hello"}]) # But we cannot add an empty column, then add a value to it. self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo2", {}]) - with self.assertRaisesRegexp(ValueError, r'Cannot change .* between formula and data'): + with self.assertRaisesRegex(ValueError, r'Cannot change .* between formula and data'): self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 1, {"foo2": "hello"}]) self.assertTableData('GristSummary_7_Address', cols="all", data=[ diff --git a/sandbox/grist/test_table_data_set.py b/sandbox/grist/test_table_data_set.py index 752b7433..e6135add 100644 --- a/sandbox/grist/test_table_data_set.py +++ b/sandbox/grist/test_table_data_set.py @@ -1,3 +1,5 @@ +import six + import actions import schema import table_data_set @@ -37,7 +39,7 @@ class TestTableDataSet(unittest.TestCase): if a.table_id not in self._table_data_set.all_tables: self._table_data_set.apply_doc_action(a) - for a in sample["SCHEMA"].itervalues(): + for a in six.itervalues(sample["SCHEMA"]): self._table_data_set.BulkAddRecord(*a) # Create AddTable actions for each table described in the metadata. @@ -61,11 +63,11 @@ class TestTableDataSet(unittest.TestCase): }) # Sort the columns in the schema according to the parentPos field from the column records. - for action in add_tables.itervalues(): + for action in six.itervalues(add_tables): action.columns.sort(key=lambda r: r["parentPos"]) self._table_data_set.AddTable(*action) - for a in sample["DATA"].itervalues(): + for a in six.itervalues(sample["DATA"]): self._table_data_set.ReplaceTableData(*a) @@ -92,11 +94,11 @@ class TestTableDataSet(unittest.TestCase): if "USE_SAMPLE" in data: expected_data = self.samples[data.pop("USE_SAMPLE")]["DATA"].copy() expected_data.update({t: testutil.table_data_from_rows(t, tdata[0], tdata[1:]) - for (t, tdata) in data.iteritems()}) + for (t, tdata) in six.iteritems(data)}) self._verify_data(expected_data) else: raise ValueError("Unrecognized step %s in test script" % step) - except Exception, e: + except Exception as e: new_args0 = "LINE %s: %s" % (line, e.args[0]) e.args = (new_args0,) + e.args[1:] raise @@ -117,7 +119,7 @@ class TestTableDataSet(unittest.TestCase): def _verify_data(self, expected_data, ignore_formulas=False): observed_data = {t: self._prep_data(*data) - for t, data in self._table_data_set.all_tables.iteritems() + for t, data in six.iteritems(self._table_data_set.all_tables) if not t.startswith("_grist_")} if ignore_formulas: observed_data = self._strip_formulas(observed_data) @@ -125,7 +127,7 @@ class TestTableDataSet(unittest.TestCase): if observed_data != expected_data: lines = [] - for table in sorted(observed_data.viewkeys() | expected_data.viewkeys()): + for table in sorted(six.viewkeys(observed_data) | six.viewkeys(expected_data)): if table not in expected_data: lines.append("*** Table %s observed but not expected\n" % table) elif table not in observed_data: @@ -141,11 +143,11 @@ class TestTableDataSet(unittest.TestCase): self.fail("\n" + "".join(lines)) def _strip_formulas(self, all_data): - return {t: self._strip_formulas_table(*data) for t, data in all_data.iteritems()} + return {t: self._strip_formulas_table(*data) for t, data in six.iteritems(all_data)} def _strip_formulas_table(self, table_id, row_ids, columns): return actions.TableData(table_id, row_ids, { - col_id: col for col_id, col in columns.iteritems() + col_id: col for col_id, col in six.iteritems(columns) if not self._table_data_set.get_col_info(table_id, col_id)["isFormula"] }) @@ -155,7 +157,7 @@ class TestTableDataSet(unittest.TestCase): return [v for r, v in sorted(zip(row_ids, col))] sorted_data = actions.TableData(table_id, sorted(row_ids), - {c: sort(col) for c, col in columns.iteritems()}) + {c: sort(col) for c, col in six.iteritems(columns)}) return actions.encode_objects(testutil.replace_nans(sorted_data)) @classmethod diff --git a/sandbox/grist/test_textbuilder.py b/sandbox/grist/test_textbuilder.py index aeb79cce..d141ec4f 100644 --- a/sandbox/grist/test_textbuilder.py +++ b/sandbox/grist/test_textbuilder.py @@ -9,7 +9,7 @@ class TestTextBuilder(unittest.TestCase): def test_validate_patch(self): text = "To be or not to be" patch = make_patch(text, 3, 8, "SEE OR") - self.assertEquals(textbuilder.validate_patch(text, patch), None) + self.assertEqual(textbuilder.validate_patch(text, patch), None) with self.assertRaises(ValueError): textbuilder.validate_patch('X' + text, patch) @@ -19,15 +19,15 @@ class TestTextBuilder(unittest.TestCase): patches = make_regexp_patches(t1.get_text(), re.compile(r'be|to', re.I), lambda m: (m.group() + m.group()).upper()) t2 = textbuilder.Replacer(t1, patches) - self.assertEquals(t2.get_text(), "TOTO BEBE or not\n TOTO BEBE?\n") - self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 0, 4, "xxx")), + self.assertEqual(t2.get_text(), "TOTO BEBE or not\n TOTO BEBE?\n") + self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 0, 4, "xxx")), (t1.get_text(), value, Patch(0, 2, "To", "xxx"))) - self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 5, 9, "xxx")), + self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 5, 9, "xxx")), (t1.get_text(), value, Patch(3, 5, "be", "xxx"))) - self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 18, 23, "xxx")), + self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 18, 23, "xxx")), (t1.get_text(), value, Patch(14, 17, " to", "xxx"))) # Match the entire second line - self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 17, 29, "xxx")), + self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 17, 29, "xxx")), (t1.get_text(), value, Patch(13, 21, " to be?", "xxx"))) def test_combiner(self): diff --git a/sandbox/grist/test_undo.py b/sandbox/grist/test_undo.py index 12414b86..76fa1681 100644 --- a/sandbox/grist/test_undo.py +++ b/sandbox/grist/test_undo.py @@ -26,7 +26,7 @@ class TestUndo(test_engine.EngineTestCase): # Now undo just the first action. The list of undo DocActions for it does not mention the # newly added column, and fails to clean it up. This would leave the doc in an inconsistent # state, and we should not allow it. - with self.assertRaisesRegexp(AssertionError, + with self.assertRaisesRegex(AssertionError, re.compile(r"Internal schema inconsistent.*'NewCol'", re.S)): self.apply_undo_actions(out_actions1.undo) @@ -40,7 +40,7 @@ class TestUndo(test_engine.EngineTestCase): # In practice it's harmless: properly calculated fields get restored correct, and the private # metadata fields get brought up-to-date when used via Record interface, which is what we do # using this assertEqual(). - self.assertEqual([[r.id, r.tableId, map(int, r.columns)] + self.assertEqual([[r.id, r.tableId, list(map(int, r.columns))] for r in self.engine.docmodel.tables.table.filter_records()], [ [1, "Students", [1,2,4,5,6]], [2, "Schools", [10,12]], @@ -73,7 +73,7 @@ class TestUndo(test_engine.EngineTestCase): # The undo failed, and data should look as before the undo. self.engine.assert_schema_consistent() - self.assertEqual([[r.id, r.tableId, map(int, r.columns)] + self.assertEqual([[r.id, r.tableId, list(map(int, r.columns))] for r in self.engine.docmodel.tables.table.filter_records()], [ [1, "Students", [1,2,4,5,6]], [2, "Schools", [10,12]], diff --git a/sandbox/grist/test_useractions.py b/sandbox/grist/test_useractions.py index d2b2227a..6d8c466e 100644 --- a/sandbox/grist/test_useractions.py +++ b/sandbox/grist/test_useractions.py @@ -689,7 +689,7 @@ class TestUserActions(test_engine.EngineTestCase): # Simple failure: bad action (last argument should be a dict). It shouldn't cause any actions # in the first place, just raise an exception about the argument being an int. - with self.assertRaisesRegexp(AttributeError, r"'int'"): + with self.assertRaisesRegex(AttributeError, r"'int'"): self.apply_user_action(['AddColumn', 'Address', "A", 17]) # Do some successful actions, just to make sure we know what they look like. @@ -699,14 +699,14 @@ class TestUserActions(test_engine.EngineTestCase): )]) # More complicated: here some actions should succeed, but get reverted when a later one fails. - with self.assertRaisesRegexp(AttributeError, r"'int'"): + with self.assertRaisesRegex(AttributeError, r"'int'"): self.engine.apply_user_actions([useractions.from_repr(ua) for ua in ( ['UpdateRecord', 'Address', 11, {"city": "New York3"}], ['AddColumn', 'Address', "C", {"isFormula": True}], ['AddColumn', 'Address', "D", 17] )]) - with self.assertRaisesRegexp(Exception, r"non-existent record #77"): + with self.assertRaisesRegex(Exception, r"non-existent record #77"): self.engine.apply_user_actions([useractions.from_repr(ua) for ua in ( ['UpdateRecord', 'Address', 11, {"city": "New York4"}], ['UpdateRecord', 'Address', 77, {"city": "Chicago"}], diff --git a/sandbox/grist/testscript.json b/sandbox/grist/testscript.json index 920ebcd1..66c3d4ac 100644 --- a/sandbox/grist/testscript.json +++ b/sandbox/grist/testscript.json @@ -454,16 +454,16 @@ // Check that Update Record on _grist_Tables_column properly triggers schema-change actions "USER_ACTIONS": [ ["UpdateRecord", "_grist_Tables_column", 3, - {"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}], + {"formula": "rec.firstName.upper() + ' ' + rec.lastName"}], ["UpdateRecord", "_grist_Tables_column", 6, {"colId" : "shortSchool"}] ], "ACTIONS": { "stored": [ ["ModifyColumn", "Students", "fullName", - {"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}], + {"formula": "rec.firstName.upper() + ' ' + rec.lastName"}], ["UpdateRecord", "_grist_Tables_column", 3, - {"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}], + {"formula": "rec.firstName.upper() + ' ' + rec.lastName"}], ["RenameColumn", "Students", "schoolShort", "shortSchool"], ["UpdateRecord", "_grist_Tables_column", 6, {"colId": "shortSchool"}], ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], @@ -1921,14 +1921,14 @@ // Check formula error handling "USER_ACTIONS" : [ ["ModifyColumn", "Students", "fullName", {"formula" : "!#@%&T#$UDSAIKVFsdhifzsk" }], - ["ModifyColumn", "Students", "schoolRegion", {"formula" : "5*len($firstName) / $fullNameLen" }] + ["ModifyColumn", "Students", "schoolRegion", {"formula" : "5*len($firstName) // $fullNameLen" }] ], "ACTIONS" : { "stored" : [ ["ModifyColumn", "Students", "fullName", {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}], ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}], - ["ModifyColumn", "Students", "schoolRegion", {"formula": "5*len($firstName) / $fullNameLen"}], - ["UpdateRecord", "_grist_Tables_column", 9, {"formula": "5*len($firstName) / $fullNameLen"}], + ["ModifyColumn", "Students", "schoolRegion", {"formula": "5*len($firstName) // $fullNameLen"}], + ["UpdateRecord", "_grist_Tables_column", 9, {"formula": "5*len($firstName) // $fullNameLen"}], ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], {"fullName" : [["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"]] @@ -2719,15 +2719,15 @@ ["APPLY", { "USER_ACTIONS": [ // Basic tests - ["EvalCode", "print 'cats'", null], - ["EvalCode", "2 * 3 - 1 / 7 + 4", null], + ["EvalCode", "print('cats')", null], + ["EvalCode", "2 * 3 - 1 // 7 + 4", null], // Exception ["EvalCode", "raise Exception('everything broke')", null], // Incomplete structure ["EvalCode", "for x in range(1, 100):", null], // Re-evaluation - ["EvalCode", "print 'cats'", 1], - ["EvalCode", "print 'dogs'", 1], + ["EvalCode", "print('cats')", 1], + ["EvalCode", "print('dogs')", 1], // Function definition ["EvalCode", "def foo(x):\n\treturn x * 10\n", null], ["EvalCode", "foo(10)", null] @@ -2735,15 +2735,15 @@ "ACTIONS" : { "stored" : [ ["AddRecord", "_grist_REPL_Hist", 1, - {"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}], + {"code": "print('cats')", "errorText": "", "outputText": "cats\n"}], ["AddRecord", "_grist_REPL_Hist", 2, - {"code": "2 * 3 - 1 / 7 + 4", "errorText": "", "outputText": "10\n"}], + {"code": "2 * 3 - 1 // 7 + 4", "errorText": "", "outputText": "10\n"}], ["AddRecord", "_grist_REPL_Hist", 3, {"code": "raise Exception('everything broke')", "errorText": "Traceback (most recent call last):\n File \"\", line 1, in \nException: everything broke\n", "outputText": ""}], ["UpdateRecord", "_grist_REPL_Hist", 1, - {"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}], + {"code": "print('cats')", "errorText": "", "outputText": "cats\n"}], ["UpdateRecord", "_grist_REPL_Hist", 1, - {"code": "print 'dogs'", "errorText": "", "outputText": "dogs\n"}], + {"code": "print('dogs')", "errorText": "", "outputText": "dogs\n"}], ["AddRecord", "_grist_REPL_Hist", 4, {"code": "def foo(x):\n\treturn x * 10\n", "errorText": "", "outputText": ""}], ["AddRecord", "_grist_REPL_Hist", 5, @@ -2755,9 +2755,9 @@ ["RemoveRecord", "_grist_REPL_Hist", 2], ["RemoveRecord", "_grist_REPL_Hist", 3], ["UpdateRecord", "_grist_REPL_Hist", 1, - {"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}], + {"code": "print('cats')", "errorText": "", "outputText": "cats\n"}], ["UpdateRecord", "_grist_REPL_Hist", 1, - {"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}], + {"code": "print('cats')", "errorText": "", "outputText": "cats\n"}], ["RemoveRecord", "_grist_REPL_Hist", 4], ["RemoveRecord", "_grist_REPL_Hist", 5] ], diff --git a/sandbox/grist/testutil.py b/sandbox/grist/testutil.py index 7a5cad0d..9a784755 100644 --- a/sandbox/grist/testutil.py +++ b/sandbox/grist/testutil.py @@ -3,6 +3,8 @@ import math import os import re +import six + import actions import logger @@ -103,7 +105,7 @@ def parse_test_sample(obj, samples={}): } data = {t: table_data_from_rows(t, data[0], data[1:]) - for t, data in obj["DATA"].iteritems()} + for t, data in six.iteritems(obj["DATA"])} return {"SCHEMA": schema, "DATA": data} diff --git a/sandbox/grist/textbuilder.py b/sandbox/grist/textbuilder.py index fc9a0e43..0fe99c0e 100644 --- a/sandbox/grist/textbuilder.py +++ b/sandbox/grist/textbuilder.py @@ -11,6 +11,8 @@ import bisect import re from collections import namedtuple +import six + Patch = namedtuple('Patch', ('start', 'end', 'old_text', 'new_text')) line_start_re = re.compile(r'^', re.M) @@ -176,4 +178,4 @@ class Combiner(Builder): offset = self._offsets[start_index - 1] part = self._parts[start_index - 1] in_patch = Patch(patch.start - offset, patch.end - offset, patch.old_text, patch.new_text) - return None if isinstance(part, basestring) else part.map_back_patch(in_patch) + return None if isinstance(part, six.string_types) else part.map_back_patch(in_patch) diff --git a/sandbox/grist/twowaymap.py b/sandbox/grist/twowaymap.py index 47e43328..3b0edaa8 100644 --- a/sandbox/grist/twowaymap.py +++ b/sandbox/grist/twowaymap.py @@ -14,6 +14,8 @@ value previously set, since the "right" dataset is "single" values), m.lookup_le that value, and m.lookup_right(value) returns a `set` of keys that map to the value. """ +import six + # Special sentinel value which can never be legitimately stored in TwoWayMap, to easily tell the # difference between a present and absent value. _NIL = object() @@ -47,6 +49,8 @@ class TwoWayMap(object): def __nonzero__(self): return bool(self._fwd) + __bool__ = __nonzero__ + def lookup_left(self, left, default=None): """ Returns the value(s) on the right corresponding to the given value on the left. """ return self._fwd.get(left, default) @@ -65,11 +69,11 @@ class TwoWayMap(object): def left_all(self): """ Returns an iterable over all values on the left.""" - return self._fwd.iterkeys() + return six.iterkeys(self._fwd) def right_all(self): """ Returns an iterable over all values on the right.""" - return self._bwd.iterkeys() + return six.iterkeys(self._bwd) def insert(self, left, right): """ Insert the (left, right) value pair. """ diff --git a/sandbox/grist/useractions.py b/sandbox/grist/useractions.py index 5fdf93f3..79f0de2a 100644 --- a/sandbox/grist/useractions.py +++ b/sandbox/grist/useractions.py @@ -4,6 +4,8 @@ import re import json import sys +import six + import acl from acl_formula import parse_acl_formula_json import actions @@ -45,7 +47,7 @@ _modify_col_schema_props = {'type', 'formula', 'isFormula'} # A few generic helpers. def select_keys(dict_obj, keys): """Return copy of dict_obj containing only the given keys.""" - return {k: v for k, v in dict_obj.iteritems() if k in keys} + return {k: v for k, v in six.iteritems(dict_obj) if k in keys} def has_value(dict_obj, key, value): """Returns True if dict_obj contains key, and its value is value.""" @@ -78,7 +80,7 @@ def useraction(method): """ Decorator for a method, which creates an action class with the same name and arguments. """ - code = method.func_code + code = method.__code__ name = method.__name__ cls = namedtuple(name, code.co_varnames[1:code.co_argcount]) setattr(_current_module, name, cls) @@ -148,7 +150,7 @@ class UserActions(object): # Map of methods implementing particular (action_name, table_id) combinations. It mirrors # global _action_method_overrides, but with methods *bound* to this UserActions instance. self._overrides = {key: method.__get__(self, UserActions) - for key, method in _action_method_overrides.iteritems()} + for key, method in six.iteritems(_action_method_overrides)} def _do_doc_action(self, action): if hasattr(action, 'simplify'): @@ -169,7 +171,7 @@ class UserActions(object): for i, row_id in enumerate(row_ids): rec = table.get_record(row_id) yield ((i, rec) if col_values is None else - (i, rec, {k: v[i] for k, v in col_values.iteritems()})) + (i, rec, {k: v[i] for k, v in six.iteritems(col_values)})) def _collect_back_references(self, table_recs): """ @@ -273,13 +275,13 @@ class UserActions(object): @useraction def AddRecord(self, table_id, row_id, column_values): return self.BulkAddRecord( - table_id, [row_id], {key: [val] for key, val in column_values.iteritems()} + table_id, [row_id], {key: [val] for key, val in six.iteritems(column_values)} )[0] @useraction def BulkAddRecord(self, table_id, row_ids, column_values): column_values = actions.decode_bulk_values(column_values) - for col_id, values in column_values.iteritems(): + for col_id, values in six.iteritems(column_values): self._ensure_column_accepts_data(table_id, col_id, values) method = self._overrides.get(('BulkAddRecord', table_id), self.doBulkAddOrReplace) return method(table_id, row_ids, column_values) @@ -339,7 +341,7 @@ class UserActions(object): def _addACLRules(self, table_id, row_ids, col_values): # Automatically populate aclFormulaParsed value by parsing aclFormula. if 'aclFormula' in col_values: - col_values['aclFormulaParsed'] = map(parse_acl_formula_json, col_values['aclFormula']) + col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']] return self.doBulkAddOrReplace(table_id, row_ids, col_values) #---------------------------------------- @@ -371,7 +373,7 @@ class UserActions(object): @useraction def UpdateRecord(self, table_id, row_id, columns): self.BulkUpdateRecord(table_id, [row_id], - {key: [col] for key, col in columns.iteritems()}) + {key: [col] for key, col in six.iteritems(columns)}) @useraction def BulkUpdateRecord(self, table_id, row_ids, columns): @@ -380,7 +382,7 @@ class UserActions(object): # Handle special tables, updates to which imply metadata actions. # Check that the update is valid. - for col_id, values in columns.iteritems(): + for col_id, values in six.iteritems(columns): self._ensure_column_accepts_data(table_id, col_id, values) # Additionally check that we are not trying to modify group-by values in a summary column @@ -413,7 +415,7 @@ class UserActions(object): @override_action('BulkUpdateRecord', '_grist_Tables') def _updateTableRecords(self, table_id, row_ids, col_values): - avoid_tableid_set = set(self._engine.tables.viewkeys()) + avoid_tableid_set = set(self._engine.tables) update_pairs = [] for i, rec, values in self._bulk_action_iter(table_id, row_ids, col_values): update_pairs.append((rec, values)) @@ -451,9 +453,9 @@ class UserActions(object): if table_renames: # Build up a dictionary mapping col_ref of each affected formula to the new formula text. formula_updates = self._prepare_formula_renames( - {(old, None): new for (old, new) in table_renames.iteritems()}) + {(old, None): new for (old, new) in six.iteritems(table_renames)}) # Add the changes to the dict of col_updates. sort for reproducible order. - for col_rec, new_formula in sorted(formula_updates.iteritems()): + for col_rec, new_formula in sorted(six.iteritems(formula_updates)): col_updates.setdefault(col_rec, {})['formula'] = new_formula # If a table changes to onDemand, any empty columns (formula columns with no set formula) @@ -464,7 +466,7 @@ class UserActions(object): for col in empty_cols: col_updates.setdefault(col, {}).update(isFormula=False, type='Text') - for col, values in col_updates.iteritems(): + for col, values in six.iteritems(col_updates): if 'type' in values: self.doModifyColumn(col.tableId, col.colId, {'type': 'Int'}) @@ -482,7 +484,7 @@ class UserActions(object): # Internal functions are used to prevent unintended additional changes from occurring. # Specifically, this prevents widgetOptions and displayCol from being cleared as a side # effect of the column type change. - for col, values in col_updates.iteritems(): + for col, values in six.iteritems(col_updates): self.doModifyColumn(col.tableId, col.colId, values) self.doBulkUpdateFromPairs('_grist_Tables_column', col_updates.items()) make_acl_updates() @@ -516,7 +518,7 @@ class UserActions(object): # Collect all renamings that we are about to apply. renames = {(c.parentId.tableId, c.colId): values['colId'] - for c, values in col_updates.iteritems() + for c, values in six.iteritems(col_updates) if has_diff_value(values, 'colId', c.colId)} if renames: @@ -524,7 +526,7 @@ class UserActions(object): formula_updates = self._prepare_formula_renames(renames) # For any affected columns, include the formula into the update. - for col_rec, new_formula in sorted(formula_updates.iteritems()): + for col_rec, new_formula in sorted(six.iteritems(formula_updates)): col_updates.setdefault(col_rec, {}).setdefault('formula', new_formula) update_pairs = col_updates.items() @@ -534,7 +536,7 @@ class UserActions(object): if col.summarySourceCol: underlying = col_updates.get(col.summarySourceCol, {}) if not all(value == getattr(col, key) or has_value(underlying, key, value) - for key, value in values.iteritems()): + for key, value in six.iteritems(values)): raise ValueError("Cannot modify summary group-by column '%s'" % col.colId) make_acl_updates = acl.prepare_acl_col_renames(self._docmodel, self, renames) @@ -576,7 +578,7 @@ class UserActions(object): def _updateACLRules(self, table_id, row_ids, col_values): # Automatically populate aclFormulaParsed value by parsing aclFormula. if 'aclFormula' in col_values: - col_values['aclFormulaParsed'] = map(parse_acl_formula_json, col_values['aclFormula']) + col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']] return self.doBulkUpdateRecord(table_id, row_ids, col_values) def _prepare_formula_renames(self, renames): @@ -605,7 +607,7 @@ class UserActions(object): # Apply the collected patches to each affected formula, converting to unicode to apply the # patches and back to byte string for how we maintain string values. result = {} - for col_rec, patches in patches_map.iteritems(): + for col_rec, patches in six.iteritems(patches_map): formula = col_rec.formula.decode('utf8') replacer = textbuilder.Replacer(textbuilder.Text(formula), patches) result[col_rec] = replacer.get_text().encode('utf8') @@ -1066,7 +1068,7 @@ class UserActions(object): # metadata record. We implement the former interface by forwarding to the latter. col = self._docmodel.get_column_rec(table_id, col_id) - update_values = {k: v for k, v in col_info.iteritems() if k in _modifiable_col_fields} + update_values = {k: v for k, v in six.iteritems(col_info) if k in _modifiable_col_fields} if '_position' in col_info: update_values['parentPos'] = col_info['_position'] self._docmodel.update([col], **update_values) @@ -1093,7 +1095,7 @@ class UserActions(object): old_col_info = schema.col_to_dict(self._engine.schema[table_id].columns[col_id], include_id=False) - col_info = {k: v for k, v in col_info.iteritems() if old_col_info.get(k, v) != v} + col_info = {k: v for k, v in six.iteritems(col_info) if old_col_info.get(k, v) != v} if not col_info: log.info("useractions.ModifyColumn is a noop") return @@ -1182,7 +1184,7 @@ class UserActions(object): # Get the values from the columns and check which have changed. all_row_ids = list(table.row_ids) - all_src_values = map(src_column.raw_get, all_row_ids) + all_src_values = [src_column.raw_get(r) for r in all_row_ids] dst_column = table.get_column(dst_col_id) changed_rows, changed_values = [], [] @@ -1230,7 +1232,7 @@ class UserActions(object): Add the given table with columns without creating views. """ # If needed, transform table_id into a valid identifier, and add a suffix to make it unique. - table_id = identifiers.pick_table_ident(table_id, avoid=self._engine.tables.viewkeys()) + table_id = identifiers.pick_table_ident(table_id, avoid=six.viewkeys(self._engine.tables)) # Sanitize and de-duplicate column identifiers. col_ids = [c['id'] for c in columns] diff --git a/sandbox/grist/usertypes.py b/sandbox/grist/usertypes.py index f903bacf..d02596a0 100644 --- a/sandbox/grist/usertypes.py +++ b/sandbox/grist/usertypes.py @@ -161,7 +161,7 @@ class Text(BaseColumnType): @classmethod def is_right_type(cls, value): - return isinstance(value, (basestring, NoneType)) + return isinstance(value, (six.string_types, NoneType)) @classmethod def typeConvert(cls, value): @@ -203,11 +203,11 @@ class Bool(BaseColumnType): # recognize. Everything else will result in alttext. if not value: return False - if isinstance(value, (float, int, long)): + if isinstance(value, (float, six.integer_types)): return True if isinstance(value, AltText): value = str(value) - if isinstance(value, basestring): + if isinstance(value, six.string_types): if value.lower() in ("false", "no", "0"): return False if value.lower() in ("true", "yes", "1"): @@ -235,7 +235,7 @@ class Int(BaseColumnType): @classmethod def is_right_type(cls, value): - return value is None or (isinstance(value, (int, long)) and not isinstance(value, bool) and + return value is None or (isinstance(value, six.integer_types) and not isinstance(value, bool) and objtypes.is_int_short(value)) @@ -252,7 +252,7 @@ class Numeric(BaseColumnType): # TODO: Python distinguishes ints from floats, while JS only has floats. A value that can be # interpreted as an int will upon being entered have type 'float', but after database reload # will have type 'int'. - return isinstance(value, (float, int, long, NoneType)) and not isinstance(value, bool) + return isinstance(value, (float, six.integer_types, NoneType)) and not isinstance(value, bool) class Date(Numeric): @@ -267,9 +267,9 @@ class Date(Numeric): return moment.dt_to_ts(value) elif isinstance(value, datetime.date): return moment.date_to_ts(value) - elif isinstance(value, (float, int, long)): + elif isinstance(value, (float, six.integer_types)): return float(value) - elif isinstance(value, basestring): + elif isinstance(value, six.string_types): # We also accept a date in ISO format (YYYY-MM-DD), the time portion is optional and ignored return moment.parse_iso_date(value) else: @@ -277,7 +277,7 @@ class Date(Numeric): @classmethod def is_right_type(cls, value): - return isinstance(value, (float, int, long, NoneType)) + return isinstance(value, (float, six.integer_types, NoneType)) @classmethod def typeConvert(cls, value, date_format, timezone='UTC'): # pylint: disable=arguments-differ @@ -306,9 +306,9 @@ class DateTime(Date): return moment.dt_to_ts(value, self.timezone) elif isinstance(value, datetime.date): return moment.date_to_ts(value, self.timezone) - elif isinstance(value, (float, int, long)): + elif isinstance(value, (float, six.integer_types)): return float(value) - elif isinstance(value, basestring): + elif isinstance(value, six.string_types): # We also accept a datetime in ISO format (YYYY-MM-DD[T]HH:mm:ss) return moment.parse_iso(value, self.timezone) else: @@ -330,7 +330,7 @@ class ChoiceList(BaseColumnType): def do_convert(self, value): if not value: return None - elif isinstance(value, basestring): + elif isinstance(value, six.string_types): # If it's a string that looks like JSON, try to parse it as such. if value.startswith('['): try: @@ -345,11 +345,11 @@ class ChoiceList(BaseColumnType): @classmethod def is_right_type(cls, value): return value is None or (isinstance(value, (tuple, list)) and - all(isinstance(item, basestring) for item in value)) + all(isinstance(item, six.string_types) for item in value)) @classmethod def typeConvert(cls, value): - if isinstance(value, basestring) and not value.startswith('['): + if isinstance(value, six.string_types) and not value.startswith('['): # Try to parse as CSV. If this doesn't work, we'll still try usual conversions later. try: tags = next(csv.reader([value])) @@ -383,7 +383,7 @@ class PositionNumber(BaseColumnType): @classmethod def is_right_type(cls, value): # Same as Numeric, but does not support None. - return isinstance(value, (float, int, long)) and not isinstance(value, bool) + return isinstance(value, (float, six.integer_types)) and not isinstance(value, bool) class ManualSortPos(PositionNumber): @@ -411,7 +411,7 @@ class Id(BaseColumnType): @classmethod def is_right_type(cls, value): - return (isinstance(value, (int, long)) and not isinstance(value, bool) and + return (isinstance(value, six.integer_types) and not isinstance(value, bool) and objtypes.is_int_short(value))