Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

(core) Simple Python 3 compatibility changes

Browse Source 
Summary: Changes that move towards python 3 compatibility that are easy to review without much thought

Test Plan: The tests

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2873
This commit is contained in:
Alex Hall 2021-06-22 17:12:25 +02:00
parent cc04c6481a
commit 16f297a250
66 changed files with 551 additions and 437 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
sandbox/gen_js_schema.py
View File

@ -24,29 +24,29 @@ def get_ts_type(col_type):
return _ts_types.get(col_type, "CellValue")
def main():
print """
print("""
/*** THIS FILE IS AUTO-GENERATED BY %s ***/
// tslint:disable:object-literal-key-quotes
export const schema = {
""" % __file__
""" % __file__)
for table in schema.schema_create_actions():
print ' "%s": {' % table.table_id
print(' "%s": {' % table.table_id)
for column in table.columns:
print ' %-20s: "%s",' % (column['id'], column['type'])
print ' },\n'
print(' %-20s: "%s",' % (column['id'], column['type']))
print(' },\n')
print """};
print("""};
export interface SchemaTypes {
"""
""")
for table in schema.schema_create_actions():
print ' "%s": {' % table.table_id
print(' "%s": {' % table.table_id)
for column in table.columns:
print ' %s: %s;' % (column['id'], get_ts_type(column['type']))
print ' };\n'
print "}"
print(' %s: %s;' % (column['id'], get_ts_type(column['type'])))
print(' };\n')
print("}")
if __name__ == '__main__':
main()

4
sandbox/grist/acl.py
View File

@ -67,7 +67,7 @@ def prepare_acl_table_renames(docmodel, useractions, table_renames_dict):
if rule_info.get("tableId") in table_renames_dict:
rule_info["tableId"] = table_renames_dict[rule_info.get("tableId")]
rule_updates.append((rule_rec, {'userAttributes': json.dumps(rule_info)}))
except Exception, e:
except Exception as e:
log.warn("Error examining aclRule: %s" % (e,))
def do_renames():
@ -103,7 +103,7 @@ def prepare_acl_col_renames(docmodel, useractions, col_renames_dict):
if new_col_id:
rule_info["lookupColId"] = new_col_id
rule_updates.append((rule_rec, {'userAttributes': json.dumps(rule_info)}))
except Exception, e:
except Exception as e:
log.warn("Error examining aclRule: %s" % (e,))
# Go through again checking if anything in ACL formulas is affected by the rename.

5
sandbox/grist/acl_formula.py
View File

@ -104,6 +104,11 @@ class _TreeConverter(ast.NodeVisitor):
return ["Const", named_constants[node.id]]
return ["Name", node.id]
def visit_Constant(self, node):
return ["Const", node.value]
visit_NameConstant = visit_Constant
def visit_Attribute(self, node):
return ["Attr", self.visit(node.value), node.attr]

12
sandbox/grist/action_obj.py
View File

@ -54,9 +54,9 @@ class ActionGroup(object):
def get_repr(self):
return {
"calc": map(actions.get_action_repr, self.calc),
"stored": map(actions.get_action_repr, self.stored),
"undo": map(actions.get_action_repr, self.undo),
"calc": [actions.get_action_repr(a) for a in self.calc],
"stored": [actions.get_action_repr(a) for a in self.stored],
"undo": [actions.get_action_repr(a) for a in self.undo],
"direct": self.direct,
"retValues": self.retValues
}
@ -64,9 +64,9 @@ class ActionGroup(object):
@classmethod
def from_json_obj(cls, data):
ag = ActionGroup()
ag.calc = map(actions.action_from_repr, data.get('calc', []))
ag.stored = map(actions.action_from_repr, data.get('stored', []))
ag.undo = map(actions.action_from_repr, data.get('undo', []))
ag.calc = [actions.action_from_repr(a) for a in data.get('calc', [])]
ag.stored = [actions.action_from_repr(a) for a in data.get('stored', [])]
ag.undo = [actions.action_from_repr(a) for a in data.get('undo', [])]
ag.retValues = data.get('retValues', [])
return ag

4
sandbox/grist/action_summary.py
View File

@ -4,6 +4,8 @@ It's used for collecting calculated values for formula columns.
"""
from collections import namedtuple
import six
import actions
from objtypes import equal_encoding
@ -76,7 +78,7 @@ class ActionSummary(object):
"""
if not column_delta:
return
full_row_ids = sorted(r for r, (before, after) in column_delta.iteritems()
full_row_ids = sorted(r for r, (before, after) in six.iteritems(column_delta)
if not equal_encoding(before, after))
defunct = is_defunct(table_id) or is_defunct(col_id)

19
sandbox/grist/actions.py
View File

@ -8,6 +8,8 @@ When communicating with Node, docActions are represented as arrays [actionName,
from collections import namedtuple
import inspect
import six
import objtypes
def _eq_with_type(self, other):
@ -86,7 +88,7 @@ def _add_simplify(SingleActionType, BulkActionType):
else:
def get_first(self):
return SingleActionType(self.table_id, self.row_ids[0],
{ key: col[0] for key, col in self.columns.iteritems()})
{ key: col[0] for key, col in six.iteritems(self.columns)})
def simplify(self):
return None if not self.row_ids else (get_first(self) if len(self.row_ids) == 1 else self)
@ -128,7 +130,7 @@ def convert_recursive_helper(converter, data):
return convert_recursive_helper(my_convert, data)
"""
if isinstance(data, dict):
return {converter(k): converter(v) for k, v in data.iteritems()}
return {converter(k): converter(v) for k, v in six.iteritems(data)}
elif isinstance(data, list):
return [converter(el) for el in data]
elif isinstance(data, tuple):
@ -142,10 +144,12 @@ def convert_action_values(converter, action):
"""
if isinstance(action, (AddRecord, UpdateRecord)):
return type(action)(action.table_id, action.row_id,
{k: converter(v) for k, v in action.columns.iteritems()})
{k: converter(v) for k, v in six.iteritems(action.columns)})
if isinstance(action, (BulkAddRecord, BulkUpdateRecord, ReplaceTableData, TableData)):
return type(action)(action.table_id, action.row_ids,
{k: map(converter, v) for k, v in action.columns.iteritems()})
return type(action)(
action.table_id, action.row_ids,
{k: [converter(value) for value in values] for k, values in six.iteritems(action.columns)}
)
return action
def convert_recursive_in_action(converter, data):
@ -173,7 +177,10 @@ def decode_bulk_values(bulk_values, decoder=objtypes.decode_object):
Decode objects in values of the form {col_id: array_of_values}, as present in bulk DocActions
and UserActions.
"""
return {k: map(decoder, v) for (k, v) in bulk_values.iteritems()}
return {
k: [decoder(value) for value in values]
for k, values in six.iteritems(bulk_values)
}
def transpose_bulk_action(bulk_action):
"""

14
sandbox/grist/autocomplete_context.py
View File

@ -4,10 +4,12 @@ Helper class for handling formula autocomplete.
It's intended to use with rlcompleter.Completer. It allows finding global names using
lowercase searches, and adds function usage information to some results.
"""
import __builtin__
from six.moves import builtins
import inspect
from collections import namedtuple
import six
# funcname is the function name, e.g. "MAX"
# argspec is the signature, e.g. "(arg, *more_args)"
# isgrist is a boolean for whether this function should be in Grist documentation.
@ -16,7 +18,7 @@ Completion = namedtuple('Completion', ['funcname', 'argspec', 'isgrist'])
def is_grist_func(func):
try:
return inspect.getmodule(func).__name__.startswith('functions.')
except Exception, e:
except Exception as e:
return e
class AutocompleteContext(object):
@ -24,7 +26,7 @@ class AutocompleteContext(object):
# rlcompleter is case-sensitive. This is hard to work around while maintaining attribute
# lookups. As a middle ground, we only introduce lowercase versions of all global names.
self._context = {
key: value for key, value in usercode_context.iteritems()
key: value for key, value in six.iteritems(usercode_context)
# Don't propose unimplemented functions in autocomplete
if not (value and callable(value) and getattr(value, 'unimplemented', None))
}
@ -32,7 +34,7 @@ class AutocompleteContext(object):
# Prepare detailed Completion objects for functions where we can supply more info.
# TODO It would be nice to include builtin functions too, but getargspec doesn't work there.
self._functions = {}
for key, value in self._context.iteritems():
for key, value in six.iteritems(self._context):
if value and callable(value):
argspec = inspect.formatargspec(*inspect.getargspec(value))
self._functions[key] = Completion(key, argspec, is_grist_func(value))
@ -47,7 +49,7 @@ class AutocompleteContext(object):
lower = key.lower()
if lower == key:
continue
if lower not in self._context and lower not in __builtin__.__dict__:
if lower not in self._context and lower not in builtins.__dict__:
self._lowercase[lower] = key
else:
# This is still good enough to find a match for, and translate back to the original.
@ -59,7 +61,7 @@ class AutocompleteContext(object):
self._lowercase[lower] = key
# Add the lowercase names to the context, and to the detailed completions in _functions.
for lower, key in self._lowercase.iteritems():
for lower, key in six.iteritems(self._lowercase):
self._context[lower] = self._context[key]
if key in self._functions:
self._functions[lower] = self._functions[key]

8
sandbox/grist/codebuilder.py
View File

@ -127,7 +127,7 @@ def _create_syntax_error_code(builder, input_text, err):
def infer(node):
try:
return next(node.infer(), None)
except astroid.exceptions.InferenceError, e:
except astroid.exceptions.InferenceError as e:
return "InferenceError on %r: %r" % (node, e)
@ -137,7 +137,7 @@ def _is_table(node):
"""
Return true if obj is a class defining a user table.
"""
return (isinstance(node, astroid.nodes.Class) and node.decorators and
return (isinstance(node, astroid.nodes.ClassDef) and node.decorators and
node.decorators.nodes[0].as_string() == 'grist.UserTable')
@ -202,7 +202,7 @@ class InferReferenceFormula(InferenceTip):
Inference helper to treat functions decorated with `grist.formulaType(grist.Reference("Foo"))`
as returning instances of table `Foo`.
"""
node_class = astroid.nodes.Function
node_class = astroid.nodes.FunctionDef
@classmethod
def filter(cls, node):
@ -320,4 +320,4 @@ def parse_grist_names(builder):
start, end = tok.startpos, tok.endpos
parsed_names.append(make_tuple(start, end, obj.name, node.arg))
return filter(None, parsed_names)
return [name for name in parsed_names if name]

4
sandbox/grist/column.py
View File

@ -2,6 +2,8 @@ import json
import types
from collections import namedtuple
import six
import depend
import objtypes
import usertypes
@ -302,7 +304,7 @@ class ChoiceListColumn(BaseColumn):
def set(self, row_id, value):
# When a JSON string is loaded, set it to a tuple parsed from it. When a list is loaded,
# convert to a tuple to keep values immutable.
if isinstance(value, basestring) and value.startswith('['):
if isinstance(value, six.string_types) and value.startswith(u'['):
try:
value = tuple(json.loads(value))
except Exception:

1
sandbox/grist/csv_patch.py
View File

@ -1,5 +1,6 @@
import re
import csv
from functools import reduce
# Monkey-patch csv.Sniffer class, in which the quote/delimiter detection has silly bugs in the
# regexp that it uses. It also seems poorly-implemented in other ways. We can probably do better

20
sandbox/grist/docactions.py
View File

@ -1,6 +1,8 @@
import six
import actions
import schema
import logger
import schema
from objtypes import strict_equal
log = logger.Logger(__name__, logger.INFO)
@ -15,7 +17,7 @@ class DocActions(object):
def AddRecord(self, table_id, row_id, column_values):
self.BulkAddRecord(
table_id, [row_id], {key: [val] for key, val in column_values.iteritems()})
table_id, [row_id], {key: [val] for key, val in six.iteritems(column_values)})
def BulkAddRecord(self, table_id, row_ids, column_values):
table = self._engine.tables[table_id]
@ -42,9 +44,9 @@ class DocActions(object):
# Collect the undo values, and unset all values in the column (i.e. set to defaults), just to
# make sure we don't have stale values hanging around.
undo_values = {}
for column in table.all_columns.itervalues():
for column in six.itervalues(table.all_columns):
if not column.is_private() and column.col_id != "id":
col_values = map(column.raw_get, row_ids)
col_values = [column.raw_get(r) for r in row_ids]
default = column.getdefault()
# If this column had all default values, don't include it into the undo BulkAddRecord.
if not all(strict_equal(val, default) for val in col_values):
@ -62,7 +64,7 @@ class DocActions(object):
def UpdateRecord(self, table_id, row_id, columns):
self.BulkUpdateRecord(
table_id, [row_id], {key: [val] for key, val in columns.iteritems()})
table_id, [row_id], {key: [val] for key, val in six.iteritems(columns)})
def BulkUpdateRecord(self, table_id, row_ids, columns):
table = self._engine.tables[table_id]
@ -72,9 +74,9 @@ class DocActions(object):
# Load the updated values.
undo_values = {}
for col_id, values in columns.iteritems():
for col_id, values in six.iteritems(columns):
col = table.get_column(col_id)
undo_values[col_id] = map(col.raw_get, row_ids)
undo_values[col_id] = [col.raw_get(r) for r in row_ids]
for (row_id, value) in zip(row_ids, values):
col.set(row_id, value)
@ -185,7 +187,7 @@ class DocActions(object):
log.info("ModifyColumn called which was a noop")
return
undo_col_info = {k: v for k, v in schema.col_to_dict(old, include_id=False).iteritems()
undo_col_info = {k: v for k, v in six.iteritems(schema.col_to_dict(old, include_id=False))
if k in col_info}
# Remove the column from the schema, then re-add it, to force creation of a new column object.
@ -249,7 +251,7 @@ class DocActions(object):
# Copy over all columns from the old table to the new.
new_table = self._engine.tables[new_table_id]
for new_column in new_table.all_columns.itervalues():
for new_column in six.itervalues(new_table.all_columns):
if not new_column.is_private():
new_column.copy_from_column(old_table.get_column(new_column.col_id))
new_table.grow_to_max() # We need to bring formula columns to the right size too.

12
sandbox/grist/docmodel.py
View File

@ -7,6 +7,8 @@ It is similar in purpose to DocModel.js on the client side.
"""
import itertools
import six
import records
import usertypes
import relabeling
@ -107,7 +109,7 @@ def enhance_model(model_class):
extras_class = getattr(MetaTableExtras, model_class.__name__, None)
if not extras_class:
return
for name, member in extras_class.__dict__.iteritems():
for name, member in six.iteritems(extras_class.__dict__):
if not name.startswith("__"):
member.__name__ = name
member.is_private = True
@ -238,7 +240,7 @@ class DocModel(object):
table_obj = record_set_or_table._table
group_by = record_set_or_table._group_by
if group_by:
values.update((k, [v] * count) for k, v in group_by.iteritems() if k not in values)
values.update((k, [v] * count) for k, v in six.iteritems(group_by) if k not in values)
else:
table_obj = record_set_or_table.table
@ -281,14 +283,14 @@ def _unify_col_values(col_values, count):
Helper that converts a dict mapping keys to values or lists of values to all lists. Non-list
values get turned into lists by repeating them count times.
"""
assert all(len(v) == count for v in col_values.itervalues() if isinstance(v, list))
assert all(len(v) == count for v in six.itervalues(col_values) if isinstance(v, list))
return {k: (v if isinstance(v, list) else [v] * count)
for k, v in col_values.iteritems()}
for k, v in six.iteritems(col_values)}
def _get_col_values_count(col_values):
"""
Helper that returns the length of the first list in among the values of col_values. If none of
the values is a list, returns 1.
"""
first_list = next((v for v in col_values.itervalues() if isinstance(v, list)), None)
first_list = next((v for v in six.itervalues(col_values) if isinstance(v, list)), None)
return len(first_list) if first_list is not None else 1

61
sandbox/grist/engine.py
View File

@ -11,6 +11,9 @@ import sys
import time
import traceback
from collections import namedtuple, OrderedDict, Hashable
import six
from six.moves import zip
from sortedcontainers import SortedSet
import acl
@ -34,8 +37,9 @@ import repl
log = logger.Logger(__name__, logger.INFO)
reload(sys)
sys.setdefaultencoding('utf8')
if six.PY2:
reload(sys)
sys.setdefaultencoding('utf8')
class OrderError(Exception):
@ -260,11 +264,11 @@ class Engine(object):
table = self.tables[data.table_id]
# Clear all columns, whether or not they are present in the data.
for column in table.all_columns.itervalues():
for column in six.itervalues(table.all_columns):
column.clear()
# Only load columns that aren't stored.
columns = {col_id: data for (col_id, data) in data.columns.iteritems()
columns = {col_id: data for (col_id, data) in six.iteritems(data.columns)
if table.has_column(col_id)}
# Add the records.
@ -296,10 +300,10 @@ class Engine(object):
table.grow_to_max()
# Load the new values.
for col_id, values in column_values.iteritems():
for col_id, values in six.iteritems(column_values):
column = table.get_column(col_id)
column.growto(growto_size)
for row_id, value in itertools.izip(row_ids, values):
for row_id, value in zip(row_ids, values):
column.set(row_id, value)
# Invalidate new records to cause the formula columns to get recomputed.
@ -314,16 +318,16 @@ class Engine(object):
query_cols = []
if query:
query_cols = [(table.get_column(col_id), values) for (col_id, values) in query.iteritems()]
query_cols = [(table.get_column(col_id), values) for (col_id, values) in six.iteritems(query)]
row_ids = [r for r in table.row_ids
if all((c.raw_get(r) in values) for (c, values) in query_cols)]
for c in table.all_columns.itervalues():
for c in six.itervalues(table.all_columns):
# pylint: disable=too-many-boolean-expressions
if ((formulas or not c.is_formula())
and (private or not c.is_private())
and c.col_id != "id" and not column.is_virtual_column(c.col_id)):
column_values[c.col_id] = map(c.raw_get, row_ids)
column_values[c.col_id] = [c.raw_get(r) for r in row_ids]
return actions.TableData(table_id, row_ids, column_values)
@ -355,8 +359,11 @@ class Engine(object):
"""
schema_actions = schema.schema_create_actions()
table_actions = [_get_table_actions(table) for table in self.docmodel.tables.all]
record_actions = [self._get_record_actions(table_id) for (table_id,t) in self.tables.iteritems()
if t.next_row_id() > 1]
record_actions = [
self._get_record_actions(table_id)
for (table_id,t) in six.iteritems(self.tables)
if t.next_row_id() > 1
]
return schema_actions + table_actions + record_actions
# Returns a BulkAddRecord action which can be used to add the currently existing data to an empty
@ -414,10 +421,10 @@ class Engine(object):
meta_tables = self.fetch_table('_grist_Tables')
meta_columns = self.fetch_table('_grist_Tables_column')
gen_schema = schema.build_schema(meta_tables, meta_columns)
gen_schema_dicts = {k: (t.tableId, dict(t.columns.iteritems()))
for k, t in gen_schema.iteritems()}
cur_schema_dicts = {k: (t.tableId, dict(t.columns.iteritems()))
for k, t in self.schema.iteritems()}
gen_schema_dicts = {k: (t.tableId, dict(t.columns))
for k, t in six.iteritems(gen_schema)}
cur_schema_dicts = {k: (t.tableId, dict(t.columns))
for k, t in six.iteritems(self.schema)}
if cur_schema_dicts != gen_schema_dicts:
import pprint
import difflib
@ -448,7 +455,7 @@ class Engine(object):
def dump_recompute_map(self):
log.debug("Recompute map (%d nodes):" % len(self.recompute_map))
for node, dirty_rows in self.recompute_map.iteritems():
for node, dirty_rows in six.iteritems(self.recompute_map):
log.debug(" Node %s: %s" % (node, dirty_rows))
@contextlib.contextmanager
@ -507,7 +514,7 @@ class Engine(object):
Called at end of _bring_all_up_to_date or _bring_lookups_up_to_date.
Issues actions for any accumulated cell changes.
"""
for node, changes in self._changes_map.iteritems():
for node, changes in six.iteritems(self._changes_map):
table = self.tables[node.table_id]
col = table.get_column(node.col_id)
# If there are changes, save them in out_actions.
@ -876,7 +883,7 @@ class Engine(object):
table = self.tables[action.table_id]
new_values = {}
extra_actions = []
for col_id, values in column_values.iteritems():
for col_id, values in six.iteritems(column_values):
col_obj = table.get_column(col_id)
values = [col_obj.convert(val) for val in values]
@ -895,7 +902,7 @@ class Engine(object):
# above does it for columns explicitly mentioned; this section does it for the other
# columns, using their default values as input to prepare_new_values().
ignore_data = isinstance(action, actions.ReplaceTableData)
for col_id, col_obj in table.all_columns.iteritems():
for col_id, col_obj in six.iteritems(table.all_columns):
if col_id in column_values or column.is_virtual_column(col_id) or col_obj.is_formula():
continue
defaults = [col_obj.getdefault() for r in row_ids]
@ -922,7 +929,7 @@ class Engine(object):
table = self.tables[action.table_id]
# Collect for each column the Column object and a list of new values.
cols = [(table.get_column(col_id), values) for (col_id, values) in column_values.iteritems()]
cols = [(table.get_column(col_id), values) for (col_id, values) in six.iteritems(column_values)]
# In comparisons below, we rely here on Python's "==" operator to check for equality. After a
# type conversion, it may compare the new type to the old, e.g. 1 == 1.0 == True. It's
@ -988,18 +995,18 @@ class Engine(object):
old_tables = self.tables
self.tables = {}
for table_id, user_table in self.gencode.usercode.__dict__.iteritems():
for table_id, user_table in six.iteritems(self.gencode.usercode.__dict__):
if isinstance(user_table, table_module.UserTable):
self.tables[table_id] = (old_tables.get(table_id) or table_module.Table(table_id, self))
# Now update the table model for each table, and tie it to its UserTable object.
for table_id, table in self.tables.iteritems():
for table_id, table in six.iteritems(self.tables):
user_table = getattr(self.gencode.usercode, table_id)
self._update_table_model(table, user_table)
user_table._set_table_impl(table)
# For any tables that are gone, use self._update_table_model to clean them up.
for table_id, table in old_tables.iteritems():
for table_id, table in six.iteritems(old_tables):
if table_id not in self.tables:
self._update_table_model(table, None)
self._repl.locals.pop(table_id, None)
@ -1032,8 +1039,8 @@ class Engine(object):
table._rebuild_model(user_table)
new_columns = table.all_columns
added_col_ids = new_columns.viewkeys() - old_columns.viewkeys()
deleted_col_ids = old_columns.viewkeys() - new_columns.viewkeys()
added_col_ids = six.viewkeys(new_columns) - six.viewkeys(old_columns)
deleted_col_ids = six.viewkeys(old_columns) - six.viewkeys(new_columns)
# Invalidate the columns that got added and anything that depends on them.
if added_col_ids:
@ -1101,7 +1108,7 @@ class Engine(object):
if self._schema_updated:
self.assert_schema_consistent()
except Exception, e:
except Exception as e:
# Save full exception info, so that we can rethrow accurately even if undo also fails.
exc_info = sys.exc_info()
# If we get an exception, we should revert all changes applied so far, to keep things
@ -1249,7 +1256,7 @@ class Engine(object):
(len_calc, len_stored, len_undo, len_ret) = checkpoint
undo_actions = self.out_actions.undo[len_undo:]
log.info("Reverting %d doc actions" % len(undo_actions))
self.user_actions.ApplyUndoActions(map(actions.get_action_repr, undo_actions))
self.user_actions.ApplyUndoActions([actions.get_action_repr(a) for a in undo_actions])
del self.out_actions.calc[len_calc:]
del self.out_actions.stored[len_stored:]
del self.out_actions.direct[len_stored:]

16
sandbox/grist/functions/__init__.py
View File

@ -1,12 +1,12 @@
# pylint: disable=wildcard-import
from date import *
from info import *
from logical import *
from lookup import *
from math import *
from stats import *
from text import *
from schedule import *
from .date import *
from .info import *
from .logical import *
from .lookup import *
from .math import *
from .stats import *
from .text import *
from .schedule import *
# Export all uppercase names, for use with `from functions import *`.
__all__ = [k for k in dir() if not k.startswith('_') and k.isupper()]

4
sandbox/grist/functions/date.py
View File

@ -1,6 +1,8 @@
import calendar
import datetime
import dateutil.parser
import six
import moment
import docmodel
@ -16,7 +18,7 @@ def _make_datetime(value):
return datetime.datetime.combine(value, datetime.time())
elif isinstance(value, datetime.time):
return datetime.datetime.combine(datetime.date.today(), value)
elif isinstance(value, basestring):
elif isinstance(value, six.string_types):
return dateutil.parser.parse(value)
else:
raise ValueError('Invalid date %r' % (value,))

4
sandbox/grist/functions/info.py
View File

@ -7,6 +7,8 @@ import math
import numbers
import re
import six
import column
from functions import date # pylint: disable=import-error
from functions.unimplemented import unimplemented
@ -217,7 +219,7 @@ def ISTEXT(value):
>>> ISTEXT(datetime.date(2011, 1, 1))
False
"""
return isinstance(value, (basestring, AltText))
return isinstance(value, (six.string_types, AltText))
# Regexp for matching email. See ISEMAIL for justification.

4
sandbox/grist/functions/logical.py
View File

@ -1,4 +1,4 @@
from info import lazy_value_or_error, is_error
from .info import lazy_value_or_error, is_error
from usertypes import AltText # pylint: disable=unused-import,import-error
@ -63,7 +63,7 @@ def IF(logical_expression, value_if_true, value_if_false):
0.0
More tests:
>>> IF(True, lambda: (1/0), lambda: (17))
>>> IF(True, lambda: (1/0), lambda: (17)) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
ZeroDivisionError: integer division or modulo by zero

17
sandbox/grist/functions/lookup.py
View File

@ -1,9 +1,10 @@
# pylint: disable=redefined-builtin, line-too-long
from collections import OrderedDict
import os
from urllib import urlencode
import urlparse
from unimplemented import unimplemented
import six
from six.moves import urllib_parse
from .unimplemented import unimplemented
@unimplemented
def ADDRESS(row, column, absolute_relative_mode, use_a1_notation, sheet):
@ -112,16 +113,16 @@ def SELF_HYPERLINK(label=None, page=None, **kwargs):
if page:
txt += "/p/{}".format(page)
if kwargs:
parts = list(urlparse.urlparse(txt))
query = OrderedDict(urlparse.parse_qsl(parts[4]))
for [key, value] in kwargs.iteritems():
parts = list(urllib_parse.urlparse(txt))
query = OrderedDict(urllib_parse.parse_qsl(parts[4]))
for [key, value] in sorted(six.iteritems(kwargs)):
key_parts = key.split('LinkKey_')
if len(key_parts) == 2 and key_parts[0] == '':
query[key_parts[1] + '_'] = value
else:
raise TypeError("unexpected keyword argument '{}' (not of form LinkKey_NAME)".format(key))
parts[4] = urlencode(query)
txt = urlparse.urlunparse(parts)
parts[4] = urllib_parse.urlencode(query)
txt = urllib_parse.urlunparse(parts)
if label:
txt = "{} {}".format(label, txt)
return txt

14
sandbox/grist/functions/math.py
View File

@ -1,12 +1,15 @@
# pylint: disable=unused-argument
from __future__ import absolute_import
import itertools
import math as _math
import operator
import os
import random
import uuid
from functools import reduce
from six.moves import zip, xrange
import six
from functions.info import ISNUMBER, ISLOGICAL
from functions.unimplemented import unimplemented
@ -358,7 +361,7 @@ def INT(value):
return int(_math.floor(value))
def _lcm(a, b):
return a * b / _gcd(a, b)
return a * b // _gcd(a, b)
def LCM(value1, *more_values):
"""
@ -790,7 +793,7 @@ def SUMPRODUCT(array1, *more_arrays):
>>> SUMPRODUCT([-0.25, -0.25], [-2, -2], [-3, -3])
-3.0
"""
return sum(reduce(operator.mul, values) for values in itertools.izip(array1, *more_arrays))
return sum(reduce(operator.mul, values) for values in zip(array1, *more_arrays))
@unimplemented
def SUMSQ(value1, value2):
@ -842,4 +845,7 @@ def TRUNC(value, places=0):
def UUID():
"""Generate a random UUID-formatted string identifier."""
return str(uuid.UUID(bytes=[chr(random.randrange(0, 256)) for _ in xrange(0, 16)], version=4))
if six.PY2:
return str(uuid.UUID(bytes=[chr(random.randrange(0, 256)) for _ in xrange(0, 16)], version=4))
else:
return str(uuid.UUID(bytes=bytes([random.randrange(0, 256) for _ in range(0, 16)]), version=4))

2
sandbox/grist/functions/schedule.py
View File

@ -1,6 +1,6 @@
from datetime import datetime, timedelta
import re
from date import DATEADD, NOW, DTIME
from .date import DATEADD, NOW, DTIME
from moment_parse import MONTH_NAMES, DAY_NAMES
# Limit exports to schedule, so that upper-case constants like MONTH_NAMES, DAY_NAMES don't end up

14
sandbox/grist/functions/stats.py
View File

@ -1,9 +1,9 @@
# pylint: disable=redefined-builtin, line-too-long, unused-argument
from math import _chain, _chain_numeric, _chain_numeric_a
from info import ISNUMBER, ISLOGICAL
from date import DATE # pylint: disable=unused-import
from unimplemented import unimplemented
from .math import _chain, _chain_numeric, _chain_numeric_a
from .info import ISNUMBER, ISLOGICAL
from .date import DATE # pylint: disable=unused-import
from .unimplemented import unimplemented
def _average(iterable):
total, count = 0.0, 0
@ -326,7 +326,7 @@ def MEDIAN(value, *more_values):
3
>>> MEDIAN(3, 5, 1, 4, 2)
3
>>> MEDIAN(xrange(10))
>>> MEDIAN(range(10))
4.5
>>> MEDIAN("Hello", "123", DATE(2015, 1, 1), 12.3)
12.3
@ -340,9 +340,9 @@ def MEDIAN(value, *more_values):
raise ValueError("MEDIAN requires at least one number")
count = len(values)
if count % 2 == 0:
return (values[count / 2 - 1] + values[count / 2]) / 2.0
return (values[count // 2 - 1] + values[count // 2]) / 2.0
else:
return values[(count - 1) / 2]
return values[(count - 1) // 2]
def MIN(value, *more_values):

46
sandbox/grist/functions/test_schedule.py
View File

@ -4,7 +4,7 @@ import timeit
import unittest
import moment
import schedule
from . import schedule
from functions.date import DTIME
from functions import date as _date
@ -68,7 +68,7 @@ class TestSchedule(unittest.TestCase):
self.assertDate(RDU(DT("2018-09-04 14:38:11"), "seconds"), "2018-09-04 14:38:11")
self.assertDate(RDU(DT("2018-09-04 14:38:11") - TICK, "seconds"), "2018-09-04 14:38:10")
with self.assertRaisesRegexp(ValueError, r"Invalid unit inches"):
with self.assertRaisesRegex(ValueError, r"Invalid unit inches"):
RDU(DT("2018-09-04 14:38:11"), "inches")
def test_round_down_to_unit_tz(self):
@ -99,11 +99,11 @@ class TestSchedule(unittest.TestCase):
self.assertEqual(schedule._parse_interval("25-months"), (25, "months"))
self.assertEqual(schedule._parse_interval("3-day"), (3, "days"))
self.assertEqual(schedule._parse_interval("2-hour"), (2, "hours"))
with self.assertRaisesRegexp(ValueError, "Not a valid interval"):
with self.assertRaisesRegex(ValueError, "Not a valid interval"):
schedule._parse_interval("1Year")
with self.assertRaisesRegexp(ValueError, "Not a valid interval"):
with self.assertRaisesRegex(ValueError, "Not a valid interval"):
schedule._parse_interval("1y")
with self.assertRaisesRegexp(ValueError, "Unknown unit"):
with self.assertRaisesRegex(ValueError, "Unknown unit"):
schedule._parse_interval("1-daily")
def test_parse_slot(self):
@ -145,41 +145,41 @@ class TestSchedule(unittest.TestCase):
def test_parse_slot_errors(self):
# Test failures with duplicate units
with self.assertRaisesRegexp(ValueError, 'Duplicate unit'):
with self.assertRaisesRegex(ValueError, 'Duplicate unit'):
schedule._parse_slot('+1d +2d', 'weeks')
with self.assertRaisesRegexp(ValueError, 'Duplicate unit'):
with self.assertRaisesRegex(ValueError, 'Duplicate unit'):
schedule._parse_slot('9:30am +2H', 'days')
with self.assertRaisesRegexp(ValueError, 'Duplicate unit'):
with self.assertRaisesRegex(ValueError, 'Duplicate unit'):
schedule._parse_slot('/15 +1d', 'months')
with self.assertRaisesRegexp(ValueError, 'Duplicate unit'):
with self.assertRaisesRegex(ValueError, 'Duplicate unit'):
schedule._parse_slot('Feb-1 12:30pm +20M', 'years')
# Test failures with improper slot types
with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'):
with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'):
schedule._parse_slot('Feb-1', 'weeks')
with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'):
with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'):
schedule._parse_slot('Monday', 'months')
with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'):
with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'):
schedule._parse_slot('4/15', 'hours')
with self.assertRaisesRegexp(ValueError, 'Invalid slot.*for unit'):
with self.assertRaisesRegex(ValueError, 'Invalid slot.*for unit'):
schedule._parse_slot('/1', 'years')
# Test failures with outright invalid slot syntax.
with self.assertRaisesRegexp(ValueError, 'Invalid slot'):
with self.assertRaisesRegex(ValueError, 'Invalid slot'):
schedule._parse_slot('Feb:1', 'weeks')
with self.assertRaisesRegexp(ValueError, 'Invalid slot'):
with self.assertRaisesRegex(ValueError, 'Invalid slot'):
schedule._parse_slot('/1d', 'months')
with self.assertRaisesRegexp(ValueError, 'Invalid slot'):
with self.assertRaisesRegex(ValueError, 'Invalid slot'):
schedule._parse_slot('10', 'hours')
with self.assertRaisesRegexp(ValueError, 'Invalid slot'):
with self.assertRaisesRegex(ValueError, 'Invalid slot'):
schedule._parse_slot('H1', 'years')
# Test failures with unknown values
with self.assertRaisesRegexp(ValueError, 'Unknown month'):
with self.assertRaisesRegex(ValueError, 'Unknown month'):
schedule._parse_slot('februarium-1', 'years')
with self.assertRaisesRegexp(ValueError, 'Unknown day of the week'):
with self.assertRaisesRegex(ValueError, 'Unknown day of the week'):
schedule._parse_slot('snu', 'weeks')
with self.assertRaisesRegexp(ValueError, 'Unknown unit'):
with self.assertRaisesRegex(ValueError, 'Unknown unit'):
schedule._parse_slot('+1t', 'hours')
def test_schedule(self):
@ -250,14 +250,14 @@ from datetime import datetime
]
self.assertEqual(timing_schedule_full(), expected_result)
t = min(timeit.repeat(stmt="t.timing_schedule_full()", setup=setup, number=N, repeat=3))
print "\n*** SCHEDULE call with 4 points: %.2f us" % (t * 1000000 / N)
print("\n*** SCHEDULE call with 4 points: %.2f us" % (t * 1000000 / N))
t = min(timeit.repeat(stmt="t.timing_schedule_init()", setup=setup, number=N, repeat=3))
print "*** Schedule constructor: %.2f us" % (t * 1000000 / N)
print("*** Schedule constructor: %.2f us" % (t * 1000000 / N))
self.assertEqual(timing_schedule_series(), expected_result)
t = min(timeit.repeat(stmt="t.timing_schedule_series()", setup=setup, number=N, repeat=3))
print "*** Schedule series with 4 points: %.2f us" % (t * 1000000 / N)
print("*** Schedule series with 4 points: %.2f us" % (t * 1000000 / N))
def timing_schedule_full():
return list(schedule.SCHEDULE("weekly: Mo 10:30am, We 10:30pm",

8
sandbox/grist/functions/text.py
View File

@ -5,7 +5,11 @@ import dateutil.parser
import numbers
import re
from unimplemented import unimplemented
import six
from six import unichr
from six.moves import xrange
from .unimplemented import unimplemented
from usertypes import AltText # pylint: disable=import-error
def CHAR(table_number):
@ -478,7 +482,7 @@ def SUBSTITUTE(text, old_text, new_text, instance_num=None):
if not old_text:
return text
if not isinstance(new_text, basestring):
if not isinstance(new_text, six.string_types):
new_text = str(new_text)
if instance_num is None:

14
sandbox/grist/gencode.py
View File

@ -19,6 +19,8 @@ import re
import imp
from collections import OrderedDict
import six
import codebuilder
from column import is_visible_column
import summary
@ -123,7 +125,7 @@ class GenCode(object):
source_table_id = summary.decode_summary_table_name(table_id)
# Sort columns by "isFormula" to output all data columns before all formula columns.
columns = sorted(table_info.columns.itervalues(), key=lambda c: c.isFormula)
columns = sorted(six.itervalues(table_info.columns), key=lambda c: c.isFormula)
if filter_for_user:
columns = [c for c in columns if is_visible_column(c.colId)]
parts = ["@grist.UserTable\nclass %s:\n" % table_id]
@ -136,9 +138,9 @@ class GenCode(object):
if summary_tables:
# Include summary formulas, for the user's information.
formulas = OrderedDict((c.colId, c) for s in summary_tables
for c in s.columns.itervalues() if c.isFormula)
for c in six.itervalues(s.columns) if c.isFormula)
parts.append(indent(textbuilder.Text("\nclass _Summary:\n")))
for col_info in formulas.itervalues():
for col_info in six.itervalues(formulas):
parts.append(indent(self._make_field(col_info, table_id), levels=2))
return textbuilder.Combiner(parts)
@ -147,7 +149,7 @@ class GenCode(object):
"""Regenerates the code text and usercode module from upated document schema."""
# Collect summary tables to group them by source table.
summary_tables = {}
for table_info in schema.itervalues():
for table_info in six.itervalues(schema):
source_table_id = summary.decode_summary_table_name(table_info.tableId)
if source_table_id:
summary_tables.setdefault(source_table_id, []).append(table_info)
@ -156,7 +158,7 @@ class GenCode(object):
"from functions import * # global uppercase functions\n" +
"import datetime, math, re # modules commonly needed in formulas\n"]
userparts = fullparts[:]
for table_info in schema.itervalues():
for table_info in six.itervalues(schema):
fullparts.append("\n\n")
fullparts.append(self._make_table_model(table_info, summary_tables.get(table_info.tableId)))
if not _is_special_table(table_info.tableId):
@ -191,5 +193,5 @@ def _is_special_table(table_id):
def exec_module_text(module_text):
# pylint: disable=exec-used
mod = imp.new_module("usercode")
exec module_text in mod.__dict__
exec(module_text, mod.__dict__)
return mod

3
sandbox/grist/gpath.py
View File

@ -1,3 +1,6 @@
from six.moves import xrange
def _is_array(obj):
return isinstance(obj, list)

3
sandbox/grist/import_actions.py
View File

@ -287,7 +287,8 @@ class ImportActions(object):
src_col_id = _import_transform_col_prefix + curr_col["colId"]
log.debug("Copying from: " + src_col_id)
column_data[curr_col["colId"]] = map(hidden_table.get_column(src_col_id).raw_get, row_ids)
src_col = hidden_table.get_column(src_col_id)
column_data[curr_col["colId"]] = [src_col.raw_get(r) for r in row_ids]
# ========= Cleanup, Prepare new table (if needed), insert data

2
sandbox/grist/imports/test_messytables.py
View File

@ -8,7 +8,7 @@ class TestMessyTables(unittest.TestCase):
def test_any_tableset(self):
path = os.path.join(os.path.dirname(__file__),
"fixtures", "nyc_schools_progress_report_ec_2013.xlsx")
with open(path, "r") as f:
with open(path, "rb") as f:
table_set = messytables.any.any_tableset(f, extension=os.path.splitext(path)[1])
self.assertIsInstance(table_set, messytables.XLSTableSet)

4
sandbox/grist/lookup.py
View File

@ -1,3 +1,5 @@
import six
import column
import depend
import records
@ -85,7 +87,7 @@ class LookupMapColumn(column.BaseColumn):
def _invalidate_affected(self, affected_keys):
# For each known relation, figure out which referring rows are affected, and invalidate them.
# The engine will notice that there have been more invalidations, and recompute things again.
for node, rel in self._lookup_relations.iteritems():
for node, rel in six.iteritems(self._lookup_relations):
affected_rows = rel.get_affected_rows_by_keys(affected_keys)
self._engine.invalidate_records(node.table_id, affected_rows, col_ids=(node.col_id,))

8
sandbox/grist/main.py
View File

@ -9,6 +9,8 @@ sys.path.append('thirdparty')
import marshal
import functools
import six
from acl_formula import parse_acl_formula
import actions
import sandbox
@ -86,7 +88,7 @@ def main():
@export
def fetch_meta_tables(formulas=True):
return {table_id: actions.get_action_repr(table_data)
for (table_id, table_data) in eng.fetch_meta_tables(formulas).iteritems()}
for (table_id, table_data) in six.iteritems(eng.fetch_meta_tables(formulas))}
@export
def load_meta_tables(meta_tables, meta_columns):
@ -100,8 +102,8 @@ def main():
@export
def create_migrations(all_tables, metadata_only=False):
doc_actions = migrations.create_migrations(
{t: table_data_from_db(t, data) for t, data in all_tables.iteritems()}, metadata_only)
return map(actions.get_action_repr, doc_actions)
{t: table_data_from_db(t, data) for t, data in six.iteritems(all_tables)}, metadata_only)
return [actions.get_action_repr(action) for action in doc_actions]
@export
def get_version():

19
sandbox/grist/migrations.py
View File

@ -1,6 +1,9 @@
import json
import re
import six
from six.moves import xrange
import actions
import identifiers
import schema
@ -56,12 +59,12 @@ def create_migrations(all_tables, metadata_only=False):
user_schema = schema.build_schema(all_tables['_grist_Tables'],
all_tables['_grist_Tables_column'],
include_builtin=False)
for t in user_schema.itervalues():
for t in six.itervalues(user_schema):
tdset.apply_doc_action(actions.AddTable(t.tableId, schema.cols_to_dict_list(t.columns)))
# For each old table/column, construct an AddTable action using the current schema.
new_schema = {a.table_id: a for a in schema.schema_create_actions()}
for table_id, data in sorted(all_tables.iteritems()):
for table_id, data in sorted(six.iteritems(all_tables)):
# User tables should already be in tdset; the rest must be metadata tables.
# (If metadata_only is true, there is simply nothing to skip here.)
if table_id not in tdset.all_tables:
@ -94,7 +97,7 @@ def get_last_migration_version():
"""
Returns the last schema version number for which we have a migration defined.
"""
return max(all_migrations.iterkeys())
return max(all_migrations)
def migration(schema_version, need_all_tables=False):
"""
@ -330,16 +333,16 @@ def migration7(tdset):
# - It doesn't fix types of Reference columns that refer to old-style summary tables
# (if the user created some such columns manually).
doc_actions = filter(None, [
doc_actions = [action for action in [
maybe_add_column(tdset, '_grist_Tables', 'summarySourceTable', 'Ref:_grist_Tables'),
maybe_add_column(tdset, '_grist_Tables_column', 'summarySourceCol', 'Ref:_grist_Tables_column')
])
] if action]
# Maps tableRef to Table object.
tables_map = {t.id: t for t in actions.transpose_bulk_action(tdset.all_tables['_grist_Tables'])}
# Maps tableName to tableRef
table_name_to_ref = {t.tableId: t.id for t in tables_map.itervalues()}
table_name_to_ref = {t.tableId: t.id for t in six.itervalues(tables_map)}
# List of Column objects
columns = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables_column']))
@ -362,14 +365,14 @@ def migration7(tdset):
# Summary tables used to be named as "Summary_<SourceName>_<ColRef1>_<ColRef2>". This regular
# expression parses that.
summary_re = re.compile(r'^Summary_(\w+?)((?:_\d+)*)$')
for t in tables_map.itervalues():
for t in six.itervalues(tables_map):
m = summary_re.match(t.tableId)
if not m or m.group(1) not in table_name_to_ref:
continue
# We have a valid summary table.
source_table_name = m.group(1)
source_table_ref = table_name_to_ref[source_table_name]
groupby_colrefs = map(int, m.group(2).strip("_").split("_"))
groupby_colrefs = [int(x) for x in m.group(2).strip("_").split("_")]
# Prepare a new-style name for the summary table. Be sure not to conflict with existing tables
# or with each other (i.e. don't rename multiple tables to the same name).
new_name = summary.encode_summary_table_name(source_table_name)

7
sandbox/grist/moment.py
View File

@ -3,10 +3,11 @@ from collections import namedtuple
import marshal
from time import time
import bisect
import itertools
import os
import moment_parse
import iso8601
import six
from six.moves import zip
# This is prepared by sandbox/install_tz.py
ZoneRecord = namedtuple("ZoneRecord", ("name", "abbrs", "offsets", "untils"))
@ -92,7 +93,7 @@ class tz(object):
self._tzinfo = tzinfo(zonelabel)
if isinstance(dt, datetime):
timestamp = dt_to_ts(dt.replace(tzinfo=self._tzinfo)) * 1000
elif isinstance(dt, (float, int, long)):
elif isinstance(dt, (float, six.integer_types)):
timestamp = dt
else:
raise TypeError("'dt' should be a datetime object or a numeric type")
@ -181,7 +182,7 @@ class Zone(object):
# "Until" times adjusted by the corresponding offsets. These are used in translating from
# datetime to absolute timestamp.
self.offset_untils = [until - offset * 60000 for (until, offset) in
itertools.izip(self.untils, self.offsets)]
zip(self.untils, self.offsets)]
# Cache of TzInfo objects for this Zone, used by get_tzinfo(). There could be multiple TzInfo
# objects, one for each possible offset, but their behavior only differs for ambiguous time.
self._tzinfo = {}

7
sandbox/grist/objtypes.py
View File

@ -17,6 +17,7 @@ from math import isnan
import moment
import records
import six
class UnmarshallableError(ValueError):
@ -186,9 +187,9 @@ def encode_object(value):
# Represent RecordSet (e.g. result of lookupRecords) in the same way as a RecordList.
return ['L'] + [encode_object(int(item)) for item in value]
elif isinstance(value, dict):
if not all(isinstance(key, basestring) for key in value):
if not all(isinstance(key, six.string_types) for key in value):
raise UnmarshallableError("Dict with non-string keys")
return ['O', {key: encode_object(val) for key, val in value.iteritems()}]
return ['O', {key: encode_object(val) for key, val in six.iteritems(value)}]
elif value == _pending_sentinel:
return ['P']
elif value == _censored_sentinel:
@ -230,7 +231,7 @@ def decode_object(value):
elif code == 'L':
return [decode_object(item) for item in args]
elif code == 'O':
return {decode_object(key): decode_object(val) for key, val in args[0].iteritems()}
return {decode_object(key): decode_object(val) for key, val in six.iteritems(args[0])}
elif code == 'P':
return _pending_sentinel
elif code == 'C':

4
sandbox/grist/records.py
View File

@ -97,6 +97,8 @@ class Record(object):
def __nonzero__(self):
return bool(self._row_id)
__bool__ = __nonzero__
def __repr__(self):
return "%s[%s]" % (self._table.table_id, self._row_id)
@ -144,6 +146,8 @@ class RecordSet(object):
def __nonzero__(self):
return bool(self._row_ids)
__bool__ = __nonzero__
def __eq__(self, other):
return (isinstance(other, RecordSet) and
(self._table, self._row_ids, self._group_by, self._sort_by) ==

7
sandbox/grist/relabeling.py
View File

@ -44,6 +44,7 @@ import itertools
import math
import struct
from six.moves import zip, xrange
from sortedcontainers import SortedList, SortedListWithKey
@ -79,7 +80,7 @@ def prepare_inserts_dumb(sortedlist, keys):
ins_groups.append((len(sortedlist), 0))
for index, ins_count in ins_groups:
adj_count = index - prev_index
adjustments.extend(itertools.izip(xrange(prev_index, index),
adjustments.extend(zip(xrange(prev_index, index),
frange_from(next_key, adj_count)))
next_key += adj_count
insertions.extend(frange_from(next_key, ins_count))
@ -233,7 +234,7 @@ class ListWithAdjustments(object):
prev_keys.sort()
new_keys = get_range(new_begin_key, new_end_key, count)
for (old_key, is_insert, i), new_key in itertools.izip(prev_keys, new_keys):
for (old_key, is_insert, i), new_key in zip(prev_keys, new_keys):
if is_insert:
self._insertions.remove(old_key)
self._insertions.add(new_key)
@ -303,7 +304,7 @@ def all_distinct(iterable):
"""
a, b = itertools.tee(iterable)
next(b, None)
return all(x != y for x, y in itertools.izip(a, b))
return all(x != y for x, y in zip(a, b))
def range_around_float(x, i):

2
sandbox/grist/repl.py
View File

@ -57,7 +57,7 @@ class REPLInterpreter(code.InteractiveInterpreter):
# like get/set attr and have that hurt us
sys.stdout = user_output
sys.stderr = user_output
exec code in self.locals
exec(code, self.locals)
except:
# bare except to catch absolutely all things the user can throw
self.showtraceback()

6
sandbox/grist/runtests.py
View File

@ -14,6 +14,8 @@ import sys
import unittest
sys.path.append('/thirdparty')
import six
def main():
# Change to the directory of this file (/grist in sandbox), to discover everything under it.
os.chdir(os.path.dirname(__file__))
@ -23,7 +25,9 @@ def main():
if "--xunit" in argv:
import xmlrunner
argv.remove("--xunit")
utf8_stdout = codecs.getwriter('utf8')(sys.stdout)
utf8_stdout = sys.stdout
if six.PY2:
utf8_stdout = codecs.getwriter('utf8')(utf8_stdout)
test_runner = xmlrunner.XMLTestRunner(stream=utf8_stdout)
if all(arg.startswith("-") for arg in argv[1:]):

4
sandbox/grist/sandbox.py
View File

@ -38,8 +38,8 @@ class Sandbox(object):
def __init__(self):
self._functions = {}
self._external_input = os.fdopen(3, "r", 64*1024)
self._external_output = os.fdopen(4, "w", 64*1024)
self._external_input = os.fdopen(3, "rb", 64*1024)
self._external_output = os.fdopen(4, "wb", 64*1024)
def _send_to_js(self, msgCode, msgBody):
# (Note that marshal version 2 is the default; we specify it explicitly for clarity. The

5
sandbox/grist/schema.py
View File

@ -10,6 +10,9 @@ Before changing this file, please review:
import itertools
from collections import OrderedDict, namedtuple
import six
import actions
SCHEMA_VERSION = 21
@ -310,7 +313,7 @@ def cols_to_dict_list(cols):
def clone_schema(schema):
return OrderedDict((t, SchemaTable(s.tableId, s.columns.copy()))
for (t, s) in schema.iteritems())
for (t, s) in six.iteritems(schema))
def build_schema(meta_tables, meta_columns, include_builtin=True):
"""

10
sandbox/grist/summary.py
View File

@ -1,6 +1,9 @@
from collections import namedtuple
import json
import re
import six
import logger
log = logger.Logger(__name__, logger.INFO)
@ -17,7 +20,7 @@ def _make_col_info(col=None, **values):
def _get_colinfo_dict(col_info, with_id=False):
"""Return a dict suitable to use with AddColumn or AddTable (when with_id=True) actions."""
col_values = {k: v for k, v in col_info._asdict().iteritems() if v is not None and k != 'colId'}
col_values = {k: v for k, v in six.iteritems(col_info._asdict()) if v is not None and k != 'colId'}
if with_id:
col_values['id'] = col_info.colId
return col_values
@ -78,9 +81,10 @@ def _update_sort_spec(sort_spec, old_table, new_table):
try:
old_sort_spec = json.loads(sort_spec)
new_sort_spec = filter(None, [adjust(col_spec) for col_spec in old_sort_spec])
new_sort_spec = [adjust(col_spec) for col_spec in old_sort_spec]
new_sort_spec = [col_spec for col_spec in new_sort_spec if col_spec]
return json.dumps(new_sort_spec, separators=(',', ':'))
except Exception, e:
except Exception:
log.warn("update_summary_section: can't parse sortColRefs JSON; clearing sortColRefs")
return ''

19
sandbox/grist/table.py
View File

@ -1,6 +1,9 @@
import collections
import types
import six
from six.moves import xrange
import column
import depend
import docmodel
@ -211,7 +214,7 @@ class Table(object):
new_cols['id'] = self._id_column
# List of Columns in the same order as they appear in the generated Model definition.
col_items = [c for c in self.Model.__dict__.iteritems() if not c[0].startswith("_")]
col_items = [c for c in six.iteritems(self.Model.__dict__) if not c[0].startswith("_")]
col_items.sort(key=lambda c: self._get_sort_order(c[1]))
for col_id, col_model in col_items:
@ -219,11 +222,11 @@ class Table(object):
new_cols[col_id] = self._create_or_update_col(col_id, col_model, default_func)
# Used for auto-completion as a record with correct properties of correct types.
self.sample_record = _make_sample_record(self.table_id, new_cols.itervalues())
self.sample_record = _make_sample_record(self.table_id, six.itervalues(new_cols))
# Note that we reuse previous special columns like lookup maps, since those not affected by
# column changes should stay the same. These get removed when unneeded using other means.
new_cols.update(sorted(self._special_cols.iteritems()))
new_cols.update(sorted(six.iteritems(self._special_cols)))
# Set the new columns.
self.all_columns = new_cols
@ -289,7 +292,7 @@ class Table(object):
"""
return ((0, col_model._creation_order)
if not isinstance(col_model, types.FunctionType) else
(1, col_model.func_code.co_firstlineno))
(1, col_model.__code__.co_firstlineno))
def next_row_id(self):
"""
@ -302,7 +305,7 @@ class Table(object):
Resizes all columns as needed so that all valid row_ids are valid indices into all columns.
"""
size = self.row_ids.max() + 1
for col_obj in self.all_columns.itervalues():
for col_obj in six.itervalues(self.all_columns):
col_obj.growto(size)
def get_column(self, col_id):
@ -325,7 +328,7 @@ class Table(object):
"""
# The tuple of keys used determines the LookupMap we need.
sort_by = kwargs.pop('sort_by', None)
col_ids = tuple(sorted(kwargs.iterkeys()))
col_ids = tuple(sorted(kwargs))
key = tuple(kwargs[c] for c in col_ids)
lookup_map = self._get_lookup_map(col_ids)
@ -383,7 +386,7 @@ class Table(object):
# TODO: It should use indices, to avoid linear searching
# TODO: It should create dependencies as needed when used from formulas.
# TODO: It should return Record instead, for convenience of user formulas
col_values = [(self.all_columns[col_id], value) for (col_id, value) in kwargs.iteritems()]
col_values = [(self.all_columns[col_id], value) for (col_id, value) in six.iteritems(kwargs)]
for row_id in self.row_ids:
if all(col.raw_get(row_id) == value for col, value in col_values):
return row_id
@ -398,7 +401,7 @@ class Table(object):
# TODO: It should use indices, to avoid linear searching
# TODO: It should create dependencies as needed when used from formulas.
# TODO: It should return Record instead, for convenience of user formulas
col_values = [(self.all_columns[col_id], value) for (col_id, value) in kwargs.iteritems()]
col_values = [(self.all_columns[col_id], value) for (col_id, value) in six.iteritems(kwargs)]
for row_id in self.row_ids:
if all(col.raw_get(row_id) == value for col, value in col_values):
yield row_id

18
sandbox/grist/table_data_set.py
View File

@ -1,4 +1,6 @@
from itertools import izip
from six.moves import zip as izip
import six
import actions
from usertypes import get_type_default
@ -29,7 +31,7 @@ class TableDataSet(object):
def apply_doc_action(self, action):
try:
getattr(self, action.__class__.__name__)(*action)
except Exception, e:
except Exception as e:
log.warn("ERROR applying action %s: %s" % (action, e))
raise
@ -48,12 +50,12 @@ class TableDataSet(object):
# Actions on records.
#----------------------------------------
def AddRecord(self, table_id, row_id, columns):
self.BulkAddRecord(table_id, [row_id], {key: [val] for key, val in columns.iteritems()})
self.BulkAddRecord(table_id, [row_id], {key: [val] for key, val in six.iteritems(columns)})
def BulkAddRecord(self, table_id, row_ids, columns):
table_data = self.all_tables[table_id]
table_data.row_ids.extend(row_ids)
for col, values in table_data.columns.iteritems():
for col, values in six.iteritems(table_data.columns):
if col in columns:
values.extend(columns[col])
else:
@ -67,19 +69,19 @@ class TableDataSet(object):
def BulkRemoveRecord(self, table_id, row_ids):
table_data = self.all_tables[table_id]
remove_set = set(row_ids)
for col, values in table_data.columns.iteritems():
for col, values in six.iteritems(table_data.columns):
values[:] = [v for r, v in izip(table_data.row_ids, values) if r not in remove_set]
table_data.row_ids[:] = [r for r in table_data.row_ids if r not in remove_set]
def UpdateRecord(self, table_id, row_id, columns):
self.BulkUpdateRecord(
table_id, [row_id], {key: [val] for key, val in columns.iteritems()})
table_id, [row_id], {key: [val] for key, val in six.iteritems(columns)})
def BulkUpdateRecord(self, table_id, row_ids, columns):
table_data = self.all_tables[table_id]
rowid_map = {r:i for i, r in enumerate(table_data.row_ids)}
table_indices = [rowid_map[r] for r in row_ids]
for col, values in columns.iteritems():
for col, values in six.iteritems(columns):
if col in table_data.columns:
col_values = table_data.columns[col]
for i, v in izip(table_indices, values):
@ -88,7 +90,7 @@ class TableDataSet(object):
def ReplaceTableData(self, table_id, row_ids, columns):
table_data = self.all_tables[table_id]
del table_data.row_ids[:]
for col, values in table_data.columns.iteritems():
for col, values in six.iteritems(table_data.columns):
del values[:]
self.BulkAddRecord(table_id, row_ids, columns)

15
sandbox/grist/test_acl_formula.py
View File

@ -5,6 +5,7 @@ import unittest
from acl_formula import parse_acl_formula
import test_engine
class TestACLFormula(unittest.TestCase):
def test_basic(self):
# Test a few basic formulas and structures, hitting everything we expect to support
@ -104,19 +105,19 @@ class TestACLFormula(unittest.TestCase):
self.assertRaises(SyntaxError, parse_acl_formula, "def foo(): pass")
# Unsupported node type
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "max(rec)")
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "user.id in {1, 2, 3}")
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "1 if user.IsAnon else 2")
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "max(rec)")
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "user.id in {1, 2, 3}")
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 if user.IsAnon else 2")
# Unsupported operation
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "1 | 2")
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "1 << 2")
self.assertRaisesRegexp(ValueError, r'Unsupported syntax', parse_acl_formula, "~test")
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 | 2")
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 << 2")
self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "~test")
# Syntax error
self.assertRaises(SyntaxError, parse_acl_formula, "[(]")
self.assertRaises(SyntaxError, parse_acl_formula, "user.id in (1,2))")
self.assertRaisesRegexp(SyntaxError, r'invalid syntax on line 1 col 9', parse_acl_formula, "foo and !bar")
self.assertRaisesRegex(SyntaxError, r'invalid syntax on line 1 col 9', parse_acl_formula, "foo and !bar")
class TestACLFormulaUserActions(test_engine.EngineTestCase):
def test_acl_actions(self):

8
sandbox/grist/test_codebuilder.py
View File

@ -2,6 +2,8 @@
import unittest
import codebuilder
import six
def make_body(formula, default=None):
return codebuilder.make_formula_body(formula, default).get_text()
@ -122,7 +124,7 @@ for a in rec:
return rec
""")
self.assertRegexpMatches(make_body(body),
self.assertRegex(make_body(body),
r"raise SyntaxError\('Grist disallows assignment" +
r" to the special variable \"rec\" on line 4 col 7'\)")
@ -135,8 +137,8 @@ return rec
self.assertEqual(make_body(u"'résumé' + $foo"), u"return 'résumé' + rec.foo")
# Check the return type of make_body()
self.assertEqual(type(make_body("foo")), unicode)
self.assertEqual(type(make_body(u"foo")), unicode)
self.assertEqual(type(make_body("foo")), six.text_type)
self.assertEqual(type(make_body(u"foo")), six.text_type)
def test_wrap_logical(self):

2
sandbox/grist/test_column_actions.py
View File

@ -338,7 +338,7 @@ class TestColumnActions(test_engine.EngineTestCase):
self.init_sample_data()
# Test that we cannot remove group-by columns from summary tables directly.
with self.assertRaisesRegexp(ValueError, "cannot remove .* group-by"):
with self.assertRaisesRegex(ValueError, "cannot remove .* group-by"):
self.apply_user_action(["BulkRemoveRecord", '_grist_Tables_column', [20,18]])
# Test that group-by columns in summary tables get removed.

4
sandbox/grist/test_docmodel.py
View File

@ -244,6 +244,6 @@ class TestDocModel(test_engine.EngineTestCase):
# Verify that positions are set such that the order is what we asked for.
student_columns = self.engine.docmodel.tables.lookupOne(tableId='Students').columns
self.assertEqual(map(int, student_columns), [1,2,4,5,6,25,22,23])
self.assertEqual(list(map(int, student_columns)), [1,2,4,5,6,25,22,23])
school_columns = self.engine.docmodel.tables.lookupOne(tableId='Schools').columns
self.assertEqual(map(int, school_columns), [24,10,12])
self.assertEqual(list(map(int, school_columns)), [24,10,12])

22
sandbox/grist/test_engine.py
View File

@ -4,6 +4,8 @@ import json
import unittest
from collections import namedtuple
import six
import actions
import column
import engine
@ -22,6 +24,8 @@ View = namedtuple('View', 'id sections')
Section = namedtuple('Section', 'id parentKey tableRef fields')
Field = namedtuple('Field', 'id colRef')
unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
class EngineTestCase(unittest.TestCase):
"""
@ -152,9 +156,9 @@ class EngineTestCase(unittest.TestCase):
# Convert observed and expected actions into a comparable form.
for k in self.action_group_action_fields:
if k in observed:
observed[k] = map(get_comparable_repr, observed[k])
observed[k] = [get_comparable_repr(v) for v in observed[k]]
if k in expected:
expected[k] = map(get_comparable_repr, expected[k])
expected[k] = [get_comparable_repr(v) for v in expected[k]]
if observed != expected:
o_lines = self._formatActionGroup(observed)
@ -192,13 +196,13 @@ class EngineTestCase(unittest.TestCase):
output = {t: self.engine.fetch_table(t) for t in self.engine.schema}
output = testutil.replace_nans(output)
output = actions.encode_objects(output)
print ''.join(self._getEngineDataLines(output))
print(''.join(self._getEngineDataLines(output)))
def dump_actions(self, out_actions):
"""
Prints out_actions in human-readable format, for help in writing / debugging tets.
"""
print "\n".join(self._formatActionGroup(out_actions.__dict__))
print("\n".join(self._formatActionGroup(out_actions.__dict__)))
def assertTableData(self, table_name, data=[], cols="all", rows="all", sort=None):
"""
@ -237,7 +241,7 @@ class EngineTestCase(unittest.TestCase):
if sort:
row_ids.sort(key=lambda r: sort(table.get_record(r)))
observed_col_data = {c.col_id: map(c.raw_get, row_ids) for c in columns if c.col_id != "id"}
observed_col_data = {c.col_id: [c.raw_get(r) for r in row_ids] for c in columns if c.col_id != "id"}
observed = actions.TableData(table_name, row_ids, observed_col_data)
self.assertEqualDocData({table_name: observed}, {table_name: expected},
col_names=col_names)
@ -286,7 +290,7 @@ class EngineTestCase(unittest.TestCase):
"""
schema = sample["SCHEMA"]
self.engine.load_meta_tables(schema['_grist_Tables'], schema['_grist_Tables_column'])
for data in sample["DATA"].itervalues():
for data in six.itervalues(sample["DATA"]):
self.engine.load_table(data)
self.engine.load_done()
@ -426,11 +430,11 @@ class TestEngine(EngineTestCase):
sample = self.samples[data.pop("USE_SAMPLE")]
expected_data = sample["DATA"].copy()
expected_data.update({t: testutil.table_data_from_rows(t, tdata[0], tdata[1:])
for (t, tdata) in data.iteritems()})
for (t, tdata) in six.iteritems(data)})
self.assertCorrectEngineData(expected_data)
else:
raise ValueError("Unrecognized step %s in test script" % step)
except Exception, e:
except Exception as e:
prefix = "LINE %s: " % line
e.args = (prefix + e.args[0],) + e.args[1:] if e.args else (prefix,)
raise
@ -526,7 +530,7 @@ class TestEngine(EngineTestCase):
# Simulate an error inside a DocAction, and make sure we restore the schema (don't leave it in
# inconsistent with metadata).
self.load_sample(testutil.parse_test_sample(self.sample1))
with self.assertRaisesRegexp(AttributeError, r"'BAD'"):
with self.assertRaisesRegex(AttributeError, r"'BAD'"):
self.add_column('Address', 'bad', isFormula=False, type="BAD")
self.engine.assert_schema_consistent()

16
sandbox/grist/test_formula_error.py
View File

@ -2,6 +2,9 @@
Tests that formula error messages (traceback) are correct
"""
import textwrap
import six
import depend
import test_engine
import testutil
@ -48,14 +51,19 @@ else:
self.assertIsInstance(exc.error, type_)
self.assertEqual(str(exc.error), message)
if tracebackRegexp:
self.assertRegexpMatches(exc.details, tracebackRegexp)
self.assertRegex(exc.details, tracebackRegexp)
def test_formula_errors(self):
self.load_sample(self.sample)
self.assertFormulaError(self.engine.get_formula_error('Math', 'excel_formula', 3),
TypeError, 'SQRT() takes exactly 1 argument (2 given)',
r"TypeError: SQRT\(\) takes exactly 1 argument \(2 given\)")
if six.PY2:
self.assertFormulaError(self.engine.get_formula_error('Math', 'excel_formula', 3),
TypeError, 'SQRT() takes exactly 1 argument (2 given)',
r"TypeError: SQRT\(\) takes exactly 1 argument \(2 given\)")
else:
self.assertFormulaError(self.engine.get_formula_error('Math', 'excel_formula', 3),
TypeError, 'SQRT() takes 1 positional argument but 2 were given',
r"TypeError: SQRT\(\) takes 1 positional argument but 2 were given")
self.assertFormulaError(self.engine.get_formula_error('Math', 'built_in_formula', 3),
TypeError, "'int' object is not iterable")

2
sandbox/grist/test_gencode.py
View File

@ -2,6 +2,8 @@ import unittest
import difflib
import re
from six.moves import xrange
import gencode
import identifiers
import records

34
sandbox/grist/test_gpath.py
View File

@ -36,10 +36,10 @@ class TestGpath(unittest.TestCase):
self.assertEqual(self.obj["hello"], "blah")
def test_set_strict(self):
with self.assertRaisesRegexp(Exception, r"non-existent"):
with self.assertRaisesRegex(Exception, r"non-existent"):
gpath.place(self.obj, ["bar", 4], 17)
with self.assertRaisesRegexp(Exception, r"not a plain object"):
with self.assertRaisesRegex(Exception, r"not a plain object"):
gpath.place(self.obj, ["foo", 0], 17)
@ -54,13 +54,13 @@ class TestGpath(unittest.TestCase):
["asdf", {"bar": 1}, {"bar": 2}, "hello", {"baz": 3}, "world"])
def test_insert_strict(self):
with self.assertRaisesRegexp(Exception, r'not an array'):
with self.assertRaisesRegex(Exception, r'not an array'):
gpath.insert(self.obj, ["foo"], "asdf")
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
with self.assertRaisesRegex(Exception, r'invalid.*index'):
gpath.insert(self.obj, ["foo", -1], 17)
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
with self.assertRaisesRegex(Exception, r'invalid.*index'):
gpath.insert(self.obj, ["foo", "foo"], 17)
def test_update(self):
@ -75,13 +75,13 @@ class TestGpath(unittest.TestCase):
def test_update_strict(self):
"""update should be strict"""
with self.assertRaisesRegexp(Exception, r'non-existent'):
with self.assertRaisesRegex(Exception, r'non-existent'):
gpath.update(self.obj, ["bar", 4], 17)
with self.assertRaisesRegexp(Exception, r'not an array'):
with self.assertRaisesRegex(Exception, r'not an array'):
gpath.update(self.obj, ["foo"], 17)
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
with self.assertRaisesRegex(Exception, r'invalid.*index'):
gpath.update(self.obj, ["foo", -1], 17)
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
with self.assertRaisesRegex(Exception, r'invalid.*index'):
gpath.update(self.obj, ["foo", None], 17)
def test_remove(self):
@ -96,13 +96,13 @@ class TestGpath(unittest.TestCase):
def test_remove_strict(self):
"""remove should be strict"""
with self.assertRaisesRegexp(Exception, r'non-existent'):
with self.assertRaisesRegex(Exception, r'non-existent'):
gpath.remove(self.obj, ["bar", 4])
with self.assertRaisesRegexp(Exception, r'not an array'):
with self.assertRaisesRegex(Exception, r'not an array'):
gpath.remove(self.obj, ["foo"])
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
with self.assertRaisesRegex(Exception, r'invalid.*index'):
gpath.remove(self.obj, ["foo", -1])
with self.assertRaisesRegexp(Exception, r'invalid.*index'):
with self.assertRaisesRegex(Exception, r'invalid.*index'):
gpath.remove(self.obj, ["foo", None])
def test_glob(self):
@ -112,7 +112,7 @@ class TestGpath(unittest.TestCase):
self.assertEqual(gpath.place(self.obj, ["foo", "*", "bar"], 17), 3)
self.assertEqual(self.obj["foo"], [{"bar": 17}, {"bar": 17}, {"baz": 3, "bar": 17}])
with self.assertRaisesRegexp(Exception, r'non-existent object at \/foo\/\*\/bad'):
with self.assertRaisesRegex(Exception, r'non-existent object at \/foo\/\*\/bad'):
gpath.place(self.obj, ["foo", "*", "bad", "test"], 10)
self.assertEqual(gpath.update(self.obj, ["foo", "*"], "hello"), 3)
@ -120,9 +120,9 @@ class TestGpath(unittest.TestCase):
def test_glob_strict_wildcard(self):
"""should only support tail wildcard for updates"""
with self.assertRaisesRegexp(Exception, r'invalid array index'):
with self.assertRaisesRegex(Exception, r'invalid array index'):
gpath.remove(self.obj, ["foo", "*"])
with self.assertRaisesRegexp(Exception, r'invalid array index'):
with self.assertRaisesRegex(Exception, r'invalid array index'):
gpath.insert(self.obj, ["foo", "*"], 1)
def test_glob_wildcard_keys(self):
@ -132,7 +132,7 @@ class TestGpath(unittest.TestCase):
self.assertEqual(gpath.place(self.obj, ["foo", 0, "*"], 17), 1)
self.assertEqual(self.obj["foo"], [{"bar": 1, '*': 17}, {"bar": 2}, {"baz": 3}])
with self.assertRaisesRegexp(Exception, r'non-existent'):
with self.assertRaisesRegex(Exception, r'non-existent'):
gpath.place(self.obj, ["*", 0, "bar"], 17)
def test_glob_nested(self):

14
sandbox/grist/test_import_actions.py
View File

@ -13,7 +13,7 @@ class TestImportActions(test_engine.EngineTestCase):
{'id': 'Zip', 'type': 'Int'}]])
self.apply_user_action(['BulkAddRecord', 'Source', [1, 2], {'Name': ['John', 'Alison'],
'City': ['New York', 'Boston'],
'Zip': [03011, 07003]}])
'Zip': [3011, 7003]}])
self.assertTableData('_grist_Tables_column', cols="subset", data=[
["id", "colId", "type", "isFormula", "formula"],
[1, "manualSort", "ManualSortPos", False, ""],
@ -79,8 +79,8 @@ class TestImportActions(test_engine.EngineTestCase):
self.assertTableData('Source', cols="all", data=[
["id", "Name", "City", "Zip", "gristHelper_Import_Name", "gristHelper_Import_City", "manualSort"],
[1, "John", "New York", 03011, "John", "New York", 1.0],
[2, "Alison", "Boston", 07003, "Alison", "Boston", 2.0],
[1, "John", "New York", 3011, "John", "New York", 1.0],
[2, "Alison", "Boston", 7003, "Alison", "Boston", 2.0],
])
self.assertPartialData("_grist_Views_section", ["id", "tableRef", 'fields'], [
@ -107,8 +107,8 @@ class TestImportActions(test_engine.EngineTestCase):
self.assertTableData('Source', cols="all", data=[
["id", "Name", "City", "Zip", "gristHelper_Import_State", "manualSort"],
[1, "John", "New York", 03011, "", 1.0],
[2, "Alison", "Boston", 07003, "", 2.0],
[1, "John", "New York", 3011, "", 1.0],
[2, "Alison", "Boston", 7003, "", 2.0],
])
self.assertPartialData("_grist_Views_section", ["id", "tableRef", 'fields'], [
[1, 1, [1, 2, 3]],
@ -168,8 +168,8 @@ class TestImportActions(test_engine.EngineTestCase):
self.assertTableData('Source', cols="all", data=[
["id", "Name", "City", "Zip", "gristHelper_Import_Name", "gristHelper_Import_City", "gristHelper_Import_Zip", "manualSort"],
[1, "John", "New York", 03011, "John", "New York", 03011, 1.0],
[2, "Alison", "Boston", 07003, "Alison", "Boston", 07003, 2.0],
[1, "John", "New York", 3011, "John", "New York", 3011, 1.0],
[2, "Alison", "Boston", 7003, "Alison", "Boston", 7003, 2.0],
])
self.assertPartialData("_grist_Views_section", ["id", "tableRef", 'fields'], [
[1, 1, [1, 2, 3]],

6
sandbox/grist/test_match_counter.py
View File

@ -3,6 +3,10 @@ import string
import timeit
import unittest
from collections import Hashable
import six
from six.moves import xrange
import match_counter
from testutil import repeat_until_passes
@ -22,7 +26,7 @@ class MatchCounterOther(object):
pass
matches = 0
for v, n in self.sample_counts.iteritems():
for v, n in six.iteritems(self.sample_counts):
if n > 0:
matches += 1
self.sample_counts[v] = 0

8
sandbox/grist/test_migrations.py
View File

@ -1,5 +1,7 @@
import unittest
import six
import actions
import schema
import table_data_set
@ -23,7 +25,7 @@ class TestMigrations(unittest.TestCase):
# Figure out the missing actions.
doc_actions = []
for table_id in sorted(current_schema.viewkeys() | migrated_schema.viewkeys()):
for table_id in sorted(six.viewkeys(current_schema) | six.viewkeys(migrated_schema)):
if table_id not in migrated_schema:
doc_actions.append(actions.AddTable(table_id, current_schema[table_id].values()))
elif table_id not in current_schema:
@ -31,7 +33,7 @@ class TestMigrations(unittest.TestCase):
else:
current_cols = current_schema[table_id]
migrated_cols = migrated_schema[table_id]
for col_id in sorted(current_cols.viewkeys() | migrated_cols.viewkeys()):
for col_id in sorted(six.viewkeys(current_cols) | six.viewkeys(migrated_cols)):
if col_id not in migrated_cols:
doc_actions.append(actions.AddColumn(table_id, col_id, current_cols[col_id]))
elif col_id not in current_cols:
@ -39,7 +41,7 @@ class TestMigrations(unittest.TestCase):
else:
current_info = current_cols[col_id]
migrated_info = migrated_cols[col_id]
delta = {k: v for k, v in current_info.iteritems() if v != migrated_info.get(k)}
delta = {k: v for k, v in six.iteritems(current_info) if v != migrated_info.get(k)}
if delta:
doc_actions.append(actions.ModifyColumn(table_id, col_id, delta))

6
sandbox/grist/test_moment.py
View File

@ -21,7 +21,7 @@ class TestMoment(unittest.TestCase):
[datetime(1979, 10, 28, 6, 0, 0), 309938400000, "EST", 300, 1, 0],
# - 2037 -
[datetime(2037, 3, 8, 6, 59, 59), 2120108399000, "EST", 300, 1, 59],
[datetime(2037, 03, 8, 7, 0, 0), 2120108400000, "EDT", 240, 3, 0],
[datetime(2037, 3, 8, 7, 0, 0), 2120108400000, "EDT", 240, 3, 0],
[datetime(2037, 11, 1, 5, 59, 59), 2140667999000, "EDT", 240, 1, 59]
]
new_york_errors = [
@ -255,7 +255,7 @@ class TestMoment(unittest.TestCase):
def test_dt_to_ds(self):
# Verify that dt_to_ts works for both naive and aware datetime objects.
value_dt = datetime(2015, 03, 14, 0, 0) # In UTC
value_dt = datetime(2015, 3, 14, 0, 0) # In UTC
value_sec = 1426291200
tzla = moment.get_zone('America/Los_Angeles')
def format_utc(ts):
@ -287,7 +287,7 @@ class TestMoment(unittest.TestCase):
self.assertEqual(value_dt_aware.strftime(fmt), '2015-02-13 20:00:00 EST')
def test_date_to_ts(self):
d = date(2015, 03, 14)
d = date(2015, 3, 14)
tzla = moment.get_zone('America/Los_Angeles')
def format_utc(ts):
return moment.ts_to_dt(ts, moment.get_zone('UTC')).strftime(fmt)

185
sandbox/grist/test_record_func.py
View File

@ -7,6 +7,7 @@ import objtypes
import testsamples
import testutil
import test_engine
from objtypes import RecordStub
log = logger.Logger(__name__, logger.INFO)
@ -18,84 +19,84 @@ class TestRecordFunc(test_engine.EngineTestCase):
def test_record_self(self):
self.load_sample(testsamples.sample_students)
self.add_column("Schools", "Foo", formula='repr(RECORD(rec))')
self.add_column("Schools", "Foo", formula='RECORD(rec)')
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"],
[2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"],
[3, "{'address': Address[13], 'id': 3, 'name': 'Yale'}"],
[4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"],
[1, {'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}],
[2, {'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}],
[3, {'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}],
[4, {'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}],
])
# A change to data is reflected
self.update_record("Schools", 3, name="UConn")
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'address': Address[11], 'id': 1, 'name': 'Columbia'}"],
[2, "{'address': Address[12], 'id': 2, 'name': 'Columbia'}"],
[3, "{'address': Address[13], 'id': 3, 'name': 'UConn'}"],
[4, "{'address': Address[14], 'id': 4, 'name': 'Yale'}"],
[1, {'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}],
[2, {'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}],
[3, {'address': RecordStub('Address', 13), 'id': 3, 'name': 'UConn'}],
[4, {'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}],
])
# A column addition is reflected
self.add_column("Schools", "Bar", formula='len($name)')
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'address': Address[11], 'Bar': 8, 'id': 1, 'name': 'Columbia'}"],
[2, "{'address': Address[12], 'Bar': 8, 'id': 2, 'name': 'Columbia'}"],
[3, "{'address': Address[13], 'Bar': 5, 'id': 3, 'name': 'UConn'}"],
[4, "{'address': Address[14], 'Bar': 4, 'id': 4, 'name': 'Yale'}"],
[1, {'address': RecordStub('Address', 11), 'Bar': 8, 'id': 1, 'name': 'Columbia'}],
[2, {'address': RecordStub('Address', 12), 'Bar': 8, 'id': 2, 'name': 'Columbia'}],
[3, {'address': RecordStub('Address', 13), 'Bar': 5, 'id': 3, 'name': 'UConn'}],
[4, {'address': RecordStub('Address', 14), 'Bar': 4, 'id': 4, 'name': 'Yale'}],
])
def test_reference(self):
self.load_sample(testsamples.sample_students)
self.add_column("Schools", "Foo", formula='repr(RECORD($address))')
self.add_column("Schools", "Foo", formula='RECORD($address)')
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'city': 'New York', 'id': 11}"],
[2, "{'city': 'Colombia', 'id': 12}"],
[3, "{'city': 'New Haven', 'id': 13}"],
[4, "{'city': 'West Haven', 'id': 14}"],
[1, {'city': 'New York', 'id': 11}],
[2, {'city': 'Colombia', 'id': 12}],
[3, {'city': 'New Haven', 'id': 13}],
[4, {'city': 'West Haven', 'id': 14}],
])
# A change to referenced data is still reflected; try a different kind of change here
self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'ciudad': 'New York', 'id': 11}"],
[2, "{'ciudad': 'Colombia', 'id': 12}"],
[3, "{'ciudad': 'New Haven', 'id': 13}"],
[4, "{'ciudad': 'West Haven', 'id': 14}"],
[1, {'ciudad': 'New York', 'id': 11}],
[2, {'ciudad': 'Colombia', 'id': 12}],
[3, {'ciudad': 'New Haven', 'id': 13}],
[4, {'ciudad': 'West Haven', 'id': 14}],
])
def test_record_expand_refs(self):
self.load_sample(testsamples.sample_students)
self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))')
self.add_column("Schools", "Foo", formula='RECORD(rec, expand_refs=1)')
self.add_column("Address", "student", type="Ref:Students")
self.update_record("Address", 12, student=6)
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'address': {'city': 'New York', 'id': 11, 'student': Students[0]}," +
" 'id': 1, 'name': 'Columbia'}"],
[2, "{'address': {'city': 'Colombia', 'id': 12, 'student': Students[6]}," +
" 'id': 2, 'name': 'Columbia'}"],
[3, "{'address': {'city': 'New Haven', 'id': 13, 'student': Students[0]}," +
" 'id': 3, 'name': 'Yale'}"],
[4, "{'address': {'city': 'West Haven', 'id': 14, 'student': Students[0]}," +
" 'id': 4, 'name': 'Yale'}"],
[1, {'address': {'city': 'New York', 'id': 11, 'student': RecordStub("Students", 0)},
'id': 1, 'name': 'Columbia'}],
[2, {'address': {'city': 'Colombia', 'id': 12, 'student': RecordStub("Students", 6)},
'id': 2, 'name': 'Columbia'}],
[3, {'address': {'city': 'New Haven', 'id': 13, 'student': RecordStub("Students", 0)},
'id': 3, 'name': 'Yale'}],
[4, {'address': {'city': 'West Haven', 'id': 14, 'student': RecordStub("Students", 0)},
'id': 4, 'name': 'Yale'}],
])
self.modify_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=2))')
self.modify_column("Schools", "Foo", formula='RECORD(rec, expand_refs=2)')
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'address': {'city': 'New York', 'id': 11, 'student': None}," +
" 'id': 1, 'name': 'Columbia'}"],
[2, "{'address': {'city': 'Colombia', 'id': 12, " +
"'student': {'firstName': 'Gerald', 'schoolName': 'Yale', 'lastName': 'Ford', " +
"'schoolCities': 'New Haven:West Haven', 'schoolIds': '3:4', 'id': 6}}," +
" 'id': 2, 'name': 'Columbia'}"],
[3, "{'address': {'city': 'New Haven', 'id': 13, 'student': None}," +
" 'id': 3, 'name': 'Yale'}"],
[4, "{'address': {'city': 'West Haven', 'id': 14, 'student': None}," +
" 'id': 4, 'name': 'Yale'}"],
[1, {'address': {'city': 'New York', 'id': 11, 'student': None},
'id': 1, 'name': 'Columbia'}],
[2, {'address': {'city': 'Colombia', 'id': 12,
'student': {'firstName': 'Gerald', 'schoolName': 'Yale', 'lastName': 'Ford',
'schoolCities': 'New Haven:West Haven', 'schoolIds': '3:4', 'id': 6}},
'id': 2, 'name': 'Columbia'}],
[3, {'address': {'city': 'New Haven', 'id': 13, 'student': None},
'id': 3, 'name': 'Yale'}],
[4, {'address': {'city': 'West Haven', 'id': 14, 'student': None},
'id': 4, 'name': 'Yale'}],
])
def test_record_date_options(self):
self.load_sample(testsamples.sample_students)
self.add_column("Schools", "Foo", formula='repr(RECORD(rec, expand_refs=1))')
self.add_column("Schools", "Foo", formula='RECORD(rec, expand_refs=1)')
self.add_column("Address", "DT", type='DateTime')
self.add_column("Address", "D", type='Date', formula="$DT and $DT.date()")
self.update_records("Address", ['id', 'DT'], [
@ -106,61 +107,57 @@ class TestRecordFunc(test_engine.EngineTestCase):
d1 = datetime.datetime(2020, 9, 13, 8, 26, 40, tzinfo=moment.tzinfo('America/New_York'))
d2 = datetime.datetime(2017, 7, 13, 22, 40, tzinfo=moment.tzinfo('America/New_York'))
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'address': {'city': 'New York', 'DT': %s, 'id': 11, 'D': %s}, " %
(repr(d1), repr(d1.date())) +
"'id': 1, 'name': 'Columbia'}"],
[2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
"'id': 2, 'name': 'Columbia'}"],
[3, "{'address': {'city': 'New Haven', 'DT': %s, 'id': 13, 'D': %s}, " %
(repr(d2), repr(d2.date())) +
"'id': 3, 'name': 'Yale'}"],
[4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
"'id': 4, 'name': 'Yale'}"],
[1, {'address': {'city': 'New York', 'DT': d1, 'id': 11, 'D': d1.date()},
'id': 1, 'name': 'Columbia'}],
[2, {'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None},
'id': 2, 'name': 'Columbia'}],
[3, {'address': {'city': 'New Haven', 'DT': d2, 'id': 13, 'D': d2.date()},
'id': 3, 'name': 'Yale'}],
[4, {'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None},
'id': 4, 'name': 'Yale'}],
])
self.modify_column("Schools", "Foo",
formula='repr(RECORD(rec, expand_refs=1, dates_as_iso=True))')
formula='RECORD(rec, expand_refs=1, dates_as_iso=True)')
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'address': {'city': 'New York', 'DT': '%s', 'id': 11, 'D': '%s'}, " %
(d1.isoformat(), d1.date().isoformat()) +
"'id': 1, 'name': 'Columbia'}"],
[2, "{'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None}, " +
"'id': 2, 'name': 'Columbia'}"],
[3, "{'address': {'city': 'New Haven', 'DT': '%s', 'id': 13, 'D': '%s'}, " %
(d2.isoformat(), d2.date().isoformat()) +
"'id': 3, 'name': 'Yale'}"],
[4, "{'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None}, " +
"'id': 4, 'name': 'Yale'}"],
[1, {'address': {'city': 'New York', 'DT': d1.isoformat(), 'id': 11, 'D': d1.date().isoformat()},
'id': 1, 'name': 'Columbia'}],
[2, {'address': {'city': 'Colombia', 'DT': None, 'id': 12, 'D': None},
'id': 2, 'name': 'Columbia'}],
[3, {'address': {'city': 'New Haven', 'DT': d2.isoformat(), 'id': 13, 'D': d2.date().isoformat()},
'id': 3, 'name': 'Yale'}],
[4, {'address': {'city': 'West Haven', 'DT': None, 'id': 14, 'D': None},
'id': 4, 'name': 'Yale'}],
])
def test_record_set(self):
self.load_sample(testsamples.sample_students)
self.add_column("Students", "schools", formula='Schools.lookupRecords(name=$schoolName)')
self.add_column("Students", "Foo", formula='repr(RECORD($schools))')
self.add_column("Students", "Foo", formula='RECORD($schools)')
self.assertPartialData("Students", ["id", "Foo"], [
[1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," +
" {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
[2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
" {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
[3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}," +
" {'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
[4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
" {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
[5, "[]"],
[6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}," +
" {'address': Address[14], 'id': 4, 'name': 'Yale'}]"],
[1, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'},
{'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}]],
[2, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'},
{'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}]],
[3, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'},
{'address': RecordStub('Address', 12), 'id': 2, 'name': 'Columbia'}]],
[4, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'},
{'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}]],
[5, []],
[6, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'},
{'address': RecordStub('Address', 14), 'id': 4, 'name': 'Yale'}]],
])
# Try a field with filtered lookupRecords result, as an iterable.
self.modify_column("Students", "Foo",
formula='repr(RECORD(s for s in $schools if s.address.city.startswith("New")))')
formula='RECORD(s for s in $schools if s.address.city.startswith("New"))')
self.assertPartialData("Students", ["id", "Foo"], [
[1, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"],
[2, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
[3, "[{'address': Address[11], 'id': 1, 'name': 'Columbia'}]"],
[4, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
[5, "[]"],
[6, "[{'address': Address[13], 'id': 3, 'name': 'Yale'}]"],
[1, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}]],
[2, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}]],
[3, [{'address': RecordStub('Address', 11), 'id': 1, 'name': 'Columbia'}]],
[4, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}]],
[5, []],
[6, [{'address': RecordStub('Address', 13), 'id': 3, 'name': 'Yale'}]],
])
def test_record_bad_calls(self):
@ -172,25 +169,25 @@ class TestRecordFunc(test_engine.EngineTestCase):
[3, objtypes.RaisedException(ValueError())],
[4, objtypes.RaisedException(ValueError())],
])
self.modify_column("Schools", "Foo", formula='repr(RECORD([rec] if $id == 2 else $id))')
self.modify_column("Schools", "Foo", formula='repr(sorted(RECORD(rec if $id == 2 else $id).items()))')
self.assertPartialData("Schools", ["id", "Foo"], [
[1, objtypes.RaisedException(ValueError())],
[2, "[{'address': Address[12], 'id': 2, 'name': 'Columbia'}]"],
[2, "[('address', Address[12]), ('id', 2), ('name', 'Columbia')]"],
[3, objtypes.RaisedException(ValueError())],
[4, objtypes.RaisedException(ValueError())],
])
self.assertEqual(self.engine.get_formula_error('Schools', 'Foo', 1).error.message,
self.assertEqual(str(self.engine.get_formula_error('Schools', 'Foo', 1).error),
'RECORD() requires a Record or an iterable of Records')
def test_record_error_cells(self):
self.load_sample(testsamples.sample_students)
self.add_column("Schools", "Foo", formula='repr(RECORD($address))')
self.add_column("Address", "Bar", formula='$id/($id%2)')
self.add_column("Schools", "Foo", formula='RECORD($address)')
self.add_column("Address", "Bar", formula='$id//($id%2)')
self.assertPartialData("Schools", ["id", "Foo"], [
[1, "{'city': 'New York', 'Bar': 11, 'id': 11}"],
[2, "{'city': 'Colombia', 'Bar': None, 'id': 12, " +
"'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"],
[3, "{'city': 'New Haven', 'Bar': 13, 'id': 13}"],
[4, "{'city': 'West Haven', 'Bar': None, 'id': 14, " +
"'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}"],
[1, {'city': 'New York', 'Bar': 11, 'id': 11}],
[2, {'city': 'Colombia', 'Bar': None, 'id': 12,
'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}],
[3, {'city': 'New Haven', 'Bar': 13, 'id': 13}],
[4, {'city': 'West Haven', 'Bar': None, 'id': 14,
'_error_': {'Bar': 'ZeroDivisionError: integer division or modulo by zero'}}],
])

13
sandbox/grist/test_relabeling.py
View File

@ -1,7 +1,8 @@
import relabeling
from sortedcontainers import SortedListWithKey
from itertools import izip
from six.moves import zip as izip, xrange
import unittest
import sys
@ -264,7 +265,7 @@ class TestRelabeling(unittest.TestCase):
slist.insert_items([(i, float('-inf')), (-i, float('inf'))])
self.assertEqual(slist.get_values(),
rev_range(2000) + [v for v,k in initial] + range(0, -2000, -1))
rev_range(2000) + [v for v,k in initial] + list(xrange(0, -2000, -1)))
#print slist.num_update_events, slist.num_updated_keys
self.assertLess(slist.avg_updated_keys(), 3)
self.assertLess(slist.num_update_events, 80)
@ -276,7 +277,7 @@ class TestRelabeling(unittest.TestCase):
slist.insert_items([(i, ins_item.key)])
# Check the end result
self.assertEqual(slist.get_values(), ['a', 'b'] + range(1000) + ['c', 'd'])
self.assertEqual(slist.get_values(), ['a', 'b'] + list(xrange(1000)) + ['c', 'd'])
self.assertAlmostEqual(slist.avg_updated_keys(), 3.5, delta=1)
self.assertLess(slist.num_update_events, 40)
@ -299,7 +300,7 @@ class TestRelabeling(unittest.TestCase):
ins_item = slist.find_value('c')
for i in xrange(1000):
slist.insert_items([(i, ins_item.key)], prepare_inserts=r.prepare_inserts_dumb)
self.assertEqual(slist.get_values(), ['a', 'b'] + range(1000) + ['c', 'd'])
self.assertEqual(slist.get_values(), ['a', 'b'] + list(xrange(1000)) + ['c', 'd'])
self.assertGreater(slist.avg_updated_keys(), 8)
def test_renumber_right_dumb(self):
@ -334,7 +335,7 @@ class TestRelabeling(unittest.TestCase):
# aL1, al0, al1, a, ar1, ar0, aR1, ...
# aL1, al0, aL2, al1, al2, a, ar2, ar1, aR2, ar0, aR1, ...
def left_half(val):
half = range(2*N - 1)
half = list(xrange(2*N - 1))
half[0::2] = ['%sL%d' % (val, i) for i in xrange(1, N + 1)]
half[1::2] = ['%sl%d' % (val, i) for i in xrange(0, N - 1)]
half[-1] = '%sl%d' % (val, N - 1)
@ -355,7 +356,7 @@ class TestRelabeling(unittest.TestCase):
def rev_range(n):
return list(reversed(range(n)))
return list(reversed(list(xrange(n))))
if __name__ == "__main__":
unittest.main()

24
sandbox/grist/test_summary2.py
View File

@ -31,7 +31,7 @@ class TestSummary2(test_engine.EngineTestCase):
self.apply_user_action(["CreateViewSection", 1, 0, "record", []])
# Check that we cannot add a non-formula column.
with self.assertRaisesRegexp(ValueError, r'non-formula column'):
with self.assertRaisesRegex(ValueError, r'non-formula column'):
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "average",
{"type": "Text", "isFormula": False}])
@ -171,7 +171,7 @@ class TestSummary2(test_engine.EngineTestCase):
# Check that we cannot rename a summary group-by column. (Perhaps it's better to raise an
# exception, but currently we translate the invalid request to a no-op.)
with self.assertRaisesRegexp(ValueError, r'Cannot modify .* group-by'):
with self.assertRaisesRegex(ValueError, r'Cannot modify .* group-by'):
self.apply_user_action(["RenameColumn", "GristSummary_7_Address", "state", "s"])
# Verify all data. We'll repeat this after renamings to make sure there are no errors.
@ -353,46 +353,46 @@ class TestSummary2(test_engine.EngineTestCase):
])
# (1) no adding/removing/renaming non-formula columns.
with self.assertRaisesRegexp(ValueError, r'non-formula column'):
with self.assertRaisesRegex(ValueError, r'non-formula column'):
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo",
{"type": "Numeric", "isFormula": False}])
with self.assertRaisesRegexp(ValueError, r'group-by column'):
with self.assertRaisesRegex(ValueError, r'group-by column'):
self.apply_user_action(["RemoveColumn", "GristSummary_7_Address", "state"])
with self.assertRaisesRegexp(ValueError, r'Cannot modify .* group-by'):
with self.assertRaisesRegex(ValueError, r'Cannot modify .* group-by'):
self.apply_user_action(["RenameColumn", "GristSummary_7_Address", "state", "st"])
# (2) no converting between formula/non-formula
with self.assertRaisesRegexp(ValueError, r'Cannot change .* formula and data'):
with self.assertRaisesRegex(ValueError, r'Cannot change .* formula and data'):
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "amount",
{"isFormula": False}])
with self.assertRaisesRegexp(ValueError, r'Cannot change .* formula and data'):
with self.assertRaisesRegex(ValueError, r'Cannot change .* formula and data'):
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "state",
{"isFormula": True}])
# (3) no editing values in non-formula columns
with self.assertRaisesRegexp(ValueError, r'Cannot enter data .* group-by'):
with self.assertRaisesRegex(ValueError, r'Cannot enter data .* group-by'):
self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 6, {"state": "ny"}])
# (4) no removing rows (this is questionable b/c empty rows might be OK to remove)
with self.assertRaisesRegexp(ValueError, r'Cannot remove record .* summary'):
with self.assertRaisesRegex(ValueError, r'Cannot remove record .* summary'):
self.apply_user_action(["RemoveRecord", "GristSummary_7_Address", 6])
# (5) no renaming summary tables.
with self.assertRaisesRegexp(ValueError, r'cannot rename .* summary'):
with self.assertRaisesRegex(ValueError, r'cannot rename .* summary'):
self.apply_user_action(["RenameTable", "GristSummary_7_Address", "GristSummary_hello"])
# Check that we can add an empty column, then set a formula for it.
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo", {}])
self.apply_user_action(["ModifyColumn", "GristSummary_7_Address", "foo", {"formula": "1+1"}])
with self.assertRaisesRegexp(ValueError, "Can't save .* to formula"):
with self.assertRaisesRegex(ValueError, "Can't save .* to formula"):
self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 1, {"foo": "hello"}])
# But we cannot add an empty column, then add a value to it.
self.apply_user_action(["AddColumn", "GristSummary_7_Address", "foo2", {}])
with self.assertRaisesRegexp(ValueError, r'Cannot change .* between formula and data'):
with self.assertRaisesRegex(ValueError, r'Cannot change .* between formula and data'):
self.apply_user_action(["UpdateRecord", "GristSummary_7_Address", 1, {"foo2": "hello"}])
self.assertTableData('GristSummary_7_Address', cols="all", data=[

22
sandbox/grist/test_table_data_set.py
View File

@ -1,3 +1,5 @@
import six
import actions
import schema
import table_data_set
@ -37,7 +39,7 @@ class TestTableDataSet(unittest.TestCase):
if a.table_id not in self._table_data_set.all_tables:
self._table_data_set.apply_doc_action(a)
for a in sample["SCHEMA"].itervalues():
for a in six.itervalues(sample["SCHEMA"]):
self._table_data_set.BulkAddRecord(*a)
# Create AddTable actions for each table described in the metadata.
@ -61,11 +63,11 @@ class TestTableDataSet(unittest.TestCase):
})
# Sort the columns in the schema according to the parentPos field from the column records.
for action in add_tables.itervalues():
for action in six.itervalues(add_tables):
action.columns.sort(key=lambda r: r["parentPos"])
self._table_data_set.AddTable(*action)
for a in sample["DATA"].itervalues():
for a in six.itervalues(sample["DATA"]):
self._table_data_set.ReplaceTableData(*a)
@ -92,11 +94,11 @@ class TestTableDataSet(unittest.TestCase):
if "USE_SAMPLE" in data:
expected_data = self.samples[data.pop("USE_SAMPLE")]["DATA"].copy()
expected_data.update({t: testutil.table_data_from_rows(t, tdata[0], tdata[1:])
for (t, tdata) in data.iteritems()})
for (t, tdata) in six.iteritems(data)})
self._verify_data(expected_data)
else:
raise ValueError("Unrecognized step %s in test script" % step)
except Exception, e:
except Exception as e:
new_args0 = "LINE %s: %s" % (line, e.args[0])
e.args = (new_args0,) + e.args[1:]
raise
@ -117,7 +119,7 @@ class TestTableDataSet(unittest.TestCase):
def _verify_data(self, expected_data, ignore_formulas=False):
observed_data = {t: self._prep_data(*data)
for t, data in self._table_data_set.all_tables.iteritems()
for t, data in six.iteritems(self._table_data_set.all_tables)
if not t.startswith("_grist_")}
if ignore_formulas:
observed_data = self._strip_formulas(observed_data)
@ -125,7 +127,7 @@ class TestTableDataSet(unittest.TestCase):
if observed_data != expected_data:
lines = []
for table in sorted(observed_data.viewkeys() | expected_data.viewkeys()):
for table in sorted(six.viewkeys(observed_data) | six.viewkeys(expected_data)):
if table not in expected_data:
lines.append("*** Table %s observed but not expected\n" % table)
elif table not in observed_data:
@ -141,11 +143,11 @@ class TestTableDataSet(unittest.TestCase):
self.fail("\n" + "".join(lines))
def _strip_formulas(self, all_data):
return {t: self._strip_formulas_table(*data) for t, data in all_data.iteritems()}
return {t: self._strip_formulas_table(*data) for t, data in six.iteritems(all_data)}
def _strip_formulas_table(self, table_id, row_ids, columns):
return actions.TableData(table_id, row_ids, {
col_id: col for col_id, col in columns.iteritems()
col_id: col for col_id, col in six.iteritems(columns)
if not self._table_data_set.get_col_info(table_id, col_id)["isFormula"]
})
@ -155,7 +157,7 @@ class TestTableDataSet(unittest.TestCase):
return [v for r, v in sorted(zip(row_ids, col))]
sorted_data = actions.TableData(table_id, sorted(row_ids),
{c: sort(col) for c, col in columns.iteritems()})
{c: sort(col) for c, col in six.iteritems(columns)})
return actions.encode_objects(testutil.replace_nans(sorted_data))
@classmethod

12
sandbox/grist/test_textbuilder.py
View File

@ -9,7 +9,7 @@ class TestTextBuilder(unittest.TestCase):
def test_validate_patch(self):
text = "To be or not to be"
patch = make_patch(text, 3, 8, "SEE OR")
self.assertEquals(textbuilder.validate_patch(text, patch), None)
self.assertEqual(textbuilder.validate_patch(text, patch), None)
with self.assertRaises(ValueError):
textbuilder.validate_patch('X' + text, patch)
@ -19,15 +19,15 @@ class TestTextBuilder(unittest.TestCase):
patches = make_regexp_patches(t1.get_text(), re.compile(r'be|to', re.I),
lambda m: (m.group() + m.group()).upper())
t2 = textbuilder.Replacer(t1, patches)
self.assertEquals(t2.get_text(), "TOTO BEBE or not\n TOTO BEBE?\n")
self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 0, 4, "xxx")),
self.assertEqual(t2.get_text(), "TOTO BEBE or not\n TOTO BEBE?\n")
self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 0, 4, "xxx")),
(t1.get_text(), value, Patch(0, 2, "To", "xxx")))
self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 5, 9, "xxx")),
self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 5, 9, "xxx")),
(t1.get_text(), value, Patch(3, 5, "be", "xxx")))
self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 18, 23, "xxx")),
self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 18, 23, "xxx")),
(t1.get_text(), value, Patch(14, 17, " to", "xxx")))
# Match the entire second line
self.assertEquals(t2.map_back_patch(make_patch(t2.get_text(), 17, 29, "xxx")),
self.assertEqual(t2.map_back_patch(make_patch(t2.get_text(), 17, 29, "xxx")),
(t1.get_text(), value, Patch(13, 21, " to be?", "xxx")))
def test_combiner(self):

6
sandbox/grist/test_undo.py
View File

@ -26,7 +26,7 @@ class TestUndo(test_engine.EngineTestCase):
# Now undo just the first action. The list of undo DocActions for it does not mention the
# newly added column, and fails to clean it up. This would leave the doc in an inconsistent
# state, and we should not allow it.
with self.assertRaisesRegexp(AssertionError,
with self.assertRaisesRegex(AssertionError,
re.compile(r"Internal schema inconsistent.*'NewCol'", re.S)):
self.apply_undo_actions(out_actions1.undo)
@ -40,7 +40,7 @@ class TestUndo(test_engine.EngineTestCase):
# In practice it's harmless: properly calculated fields get restored correct, and the private
# metadata fields get brought up-to-date when used via Record interface, which is what we do
# using this assertEqual().
self.assertEqual([[r.id, r.tableId, map(int, r.columns)]
self.assertEqual([[r.id, r.tableId, list(map(int, r.columns))]
for r in self.engine.docmodel.tables.table.filter_records()], [
[1, "Students", [1,2,4,5,6]],
[2, "Schools", [10,12]],
@ -73,7 +73,7 @@ class TestUndo(test_engine.EngineTestCase):
# The undo failed, and data should look as before the undo.
self.engine.assert_schema_consistent()
self.assertEqual([[r.id, r.tableId, map(int, r.columns)]
self.assertEqual([[r.id, r.tableId, list(map(int, r.columns))]
for r in self.engine.docmodel.tables.table.filter_records()], [
[1, "Students", [1,2,4,5,6]],
[2, "Schools", [10,12]],

6
sandbox/grist/test_useractions.py
View File

@ -689,7 +689,7 @@ class TestUserActions(test_engine.EngineTestCase):
# Simple failure: bad action (last argument should be a dict). It shouldn't cause any actions
# in the first place, just raise an exception about the argument being an int.
with self.assertRaisesRegexp(AttributeError, r"'int'"):
with self.assertRaisesRegex(AttributeError, r"'int'"):
self.apply_user_action(['AddColumn', 'Address', "A", 17])
# Do some successful actions, just to make sure we know what they look like.
@ -699,14 +699,14 @@ class TestUserActions(test_engine.EngineTestCase):
)])
# More complicated: here some actions should succeed, but get reverted when a later one fails.
with self.assertRaisesRegexp(AttributeError, r"'int'"):
with self.assertRaisesRegex(AttributeError, r"'int'"):
self.engine.apply_user_actions([useractions.from_repr(ua) for ua in (
['UpdateRecord', 'Address', 11, {"city": "New York3"}],
['AddColumn', 'Address', "C", {"isFormula": True}],
['AddColumn', 'Address', "D", 17]
)])
with self.assertRaisesRegexp(Exception, r"non-existent record #77"):
with self.assertRaisesRegex(Exception, r"non-existent record #77"):
self.engine.apply_user_actions([useractions.from_repr(ua) for ua in (
['UpdateRecord', 'Address', 11, {"city": "New York4"}],
['UpdateRecord', 'Address', 77, {"city": "Chicago"}],

32
sandbox/grist/testscript.json
View File

@ -454,16 +454,16 @@
// Check that Update Record on _grist_Tables_column properly triggers schema-change actions
"USER_ACTIONS": [
["UpdateRecord", "_grist_Tables_column", 3,
{"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}],
{"formula": "rec.firstName.upper() + ' ' + rec.lastName"}],
["UpdateRecord", "_grist_Tables_column", 6,
{"colId" : "shortSchool"}]
],
"ACTIONS": {
"stored": [
["ModifyColumn", "Students", "fullName",
{"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}],
{"formula": "rec.firstName.upper() + ' ' + rec.lastName"}],
["UpdateRecord", "_grist_Tables_column", 3,
{"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}],
{"formula": "rec.firstName.upper() + ' ' + rec.lastName"}],
["RenameColumn", "Students", "schoolShort", "shortSchool"],
["UpdateRecord", "_grist_Tables_column", 6, {"colId": "shortSchool"}],
["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8],
@ -1921,14 +1921,14 @@
// Check formula error handling
"USER_ACTIONS" : [
["ModifyColumn", "Students", "fullName", {"formula" : "!#@%&T#$UDSAIKVFsdhifzsk" }],
["ModifyColumn", "Students", "schoolRegion", {"formula" : "5*len($firstName) / $fullNameLen" }]
["ModifyColumn", "Students", "schoolRegion", {"formula" : "5*len($firstName) // $fullNameLen" }]
],
"ACTIONS" : {
"stored" : [
["ModifyColumn", "Students", "fullName", {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}],
["UpdateRecord", "_grist_Tables_column", 3, {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}],
["ModifyColumn", "Students", "schoolRegion", {"formula": "5*len($firstName) / $fullNameLen"}],
["UpdateRecord", "_grist_Tables_column", 9, {"formula": "5*len($firstName) / $fullNameLen"}],
["ModifyColumn", "Students", "schoolRegion", {"formula": "5*len($firstName) // $fullNameLen"}],
["UpdateRecord", "_grist_Tables_column", 9, {"formula": "5*len($firstName) // $fullNameLen"}],
["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8],
{"fullName" : [["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"],
["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"]]
@ -2719,15 +2719,15 @@
["APPLY", {
"USER_ACTIONS": [
// Basic tests
["EvalCode", "print 'cats'", null],
["EvalCode", "2 * 3 - 1 / 7 + 4", null],
["EvalCode", "print('cats')", null],
["EvalCode", "2 * 3 - 1 // 7 + 4", null],
// Exception
["EvalCode", "raise Exception('everything broke')", null],
// Incomplete structure
["EvalCode", "for x in range(1, 100):", null],
// Re-evaluation
["EvalCode", "print 'cats'", 1],
["EvalCode", "print 'dogs'", 1],
["EvalCode", "print('cats')", 1],
["EvalCode", "print('dogs')", 1],
// Function definition
["EvalCode", "def foo(x):\n\treturn x * 10\n", null],
["EvalCode", "foo(10)", null]
@ -2735,15 +2735,15 @@
"ACTIONS" : {
"stored" : [
["AddRecord", "_grist_REPL_Hist", 1,
{"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}],
{"code": "print('cats')", "errorText": "", "outputText": "cats\n"}],
["AddRecord", "_grist_REPL_Hist", 2,
{"code": "2 * 3 - 1 / 7 + 4", "errorText": "", "outputText": "10\n"}],
{"code": "2 * 3 - 1 // 7 + 4", "errorText": "", "outputText": "10\n"}],
["AddRecord", "_grist_REPL_Hist", 3,
{"code": "raise Exception('everything broke')", "errorText": "Traceback (most recent call last):\n File \"<input>\", line 1, in <module>\nException: everything broke\n", "outputText": ""}],
["UpdateRecord", "_grist_REPL_Hist", 1,
{"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}],
{"code": "print('cats')", "errorText": "", "outputText": "cats\n"}],
["UpdateRecord", "_grist_REPL_Hist", 1,
{"code": "print 'dogs'", "errorText": "", "outputText": "dogs\n"}],
{"code": "print('dogs')", "errorText": "", "outputText": "dogs\n"}],
["AddRecord", "_grist_REPL_Hist", 4,
{"code": "def foo(x):\n\treturn x * 10\n", "errorText": "", "outputText": ""}],
["AddRecord", "_grist_REPL_Hist", 5,
@ -2755,9 +2755,9 @@
["RemoveRecord", "_grist_REPL_Hist", 2],
["RemoveRecord", "_grist_REPL_Hist", 3],
["UpdateRecord", "_grist_REPL_Hist", 1,
{"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}],
{"code": "print('cats')", "errorText": "", "outputText": "cats\n"}],
["UpdateRecord", "_grist_REPL_Hist", 1,
{"code": "print 'cats'", "errorText": "", "outputText": "cats\n"}],
{"code": "print('cats')", "errorText": "", "outputText": "cats\n"}],
["RemoveRecord", "_grist_REPL_Hist", 4],
["RemoveRecord", "_grist_REPL_Hist", 5]
],

4
sandbox/grist/testutil.py
View File

@ -3,6 +3,8 @@ import math
import os
import re
import six
import actions
import logger
@ -103,7 +105,7 @@ def parse_test_sample(obj, samples={}):
}
data = {t: table_data_from_rows(t, data[0], data[1:])
for t, data in obj["DATA"].iteritems()}
for t, data in six.iteritems(obj["DATA"])}
return {"SCHEMA": schema, "DATA": data}

4
sandbox/grist/textbuilder.py
View File

@ -11,6 +11,8 @@ import bisect
import re
from collections import namedtuple
import six
Patch = namedtuple('Patch', ('start', 'end', 'old_text', 'new_text'))
line_start_re = re.compile(r'^', re.M)
@ -176,4 +178,4 @@ class Combiner(Builder):
offset = self._offsets[start_index - 1]
part = self._parts[start_index - 1]
in_patch = Patch(patch.start - offset, patch.end - offset, patch.old_text, patch.new_text)
return None if isinstance(part, basestring) else part.map_back_patch(in_patch)
return None if isinstance(part, six.string_types) else part.map_back_patch(in_patch)

8
sandbox/grist/twowaymap.py
View File

@ -14,6 +14,8 @@ value previously set, since the "right" dataset is "single" values), m.lookup_le
that value, and m.lookup_right(value) returns a `set` of keys that map to the value.
"""
import six
# Special sentinel value which can never be legitimately stored in TwoWayMap, to easily tell the
# difference between a present and absent value.
_NIL = object()
@ -47,6 +49,8 @@ class TwoWayMap(object):
def __nonzero__(self):
return bool(self._fwd)
__bool__ = __nonzero__
def lookup_left(self, left, default=None):
""" Returns the value(s) on the right corresponding to the given value on the left. """
return self._fwd.get(left, default)
@ -65,11 +69,11 @@ class TwoWayMap(object):
def left_all(self):
""" Returns an iterable over all values on the left."""
return self._fwd.iterkeys()
return six.iterkeys(self._fwd)
def right_all(self):
""" Returns an iterable over all values on the right."""
return self._bwd.iterkeys()
return six.iterkeys(self._bwd)
def insert(self, left, right):
""" Insert the (left, right) value pair. """

48
sandbox/grist/useractions.py
View File

@ -4,6 +4,8 @@ import re
import json
import sys
import six
import acl
from acl_formula import parse_acl_formula_json
import actions
@ -45,7 +47,7 @@ _modify_col_schema_props = {'type', 'formula', 'isFormula'}
# A few generic helpers.
def select_keys(dict_obj, keys):
"""Return copy of dict_obj containing only the given keys."""
return {k: v for k, v in dict_obj.iteritems() if k in keys}
return {k: v for k, v in six.iteritems(dict_obj) if k in keys}
def has_value(dict_obj, key, value):
"""Returns True if dict_obj contains key, and its value is value."""
@ -78,7 +80,7 @@ def useraction(method):
"""
Decorator for a method, which creates an action class with the same name and arguments.
"""
code = method.func_code
code = method.__code__
name = method.__name__
cls = namedtuple(name, code.co_varnames[1:code.co_argcount])
setattr(_current_module, name, cls)
@ -148,7 +150,7 @@ class UserActions(object):
# Map of methods implementing particular (action_name, table_id) combinations. It mirrors
# global _action_method_overrides, but with methods *bound* to this UserActions instance.
self._overrides = {key: method.__get__(self, UserActions)
for key, method in _action_method_overrides.iteritems()}
for key, method in six.iteritems(_action_method_overrides)}
def _do_doc_action(self, action):
if hasattr(action, 'simplify'):
@ -169,7 +171,7 @@ class UserActions(object):
for i, row_id in enumerate(row_ids):
rec = table.get_record(row_id)
yield ((i, rec) if col_values is None else
(i, rec, {k: v[i] for k, v in col_values.iteritems()}))
(i, rec, {k: v[i] for k, v in six.iteritems(col_values)}))
def _collect_back_references(self, table_recs):
"""
@ -273,13 +275,13 @@ class UserActions(object):
@useraction
def AddRecord(self, table_id, row_id, column_values):
return self.BulkAddRecord(
table_id, [row_id], {key: [val] for key, val in column_values.iteritems()}
table_id, [row_id], {key: [val] for key, val in six.iteritems(column_values)}
)[0]
@useraction
def BulkAddRecord(self, table_id, row_ids, column_values):
column_values = actions.decode_bulk_values(column_values)
for col_id, values in column_values.iteritems():
for col_id, values in six.iteritems(column_values):
self._ensure_column_accepts_data(table_id, col_id, values)
method = self._overrides.get(('BulkAddRecord', table_id), self.doBulkAddOrReplace)
return method(table_id, row_ids, column_values)
@ -339,7 +341,7 @@ class UserActions(object):
def _addACLRules(self, table_id, row_ids, col_values):
# Automatically populate aclFormulaParsed value by parsing aclFormula.
if 'aclFormula' in col_values:
col_values['aclFormulaParsed'] = map(parse_acl_formula_json, col_values['aclFormula'])
col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']]
return self.doBulkAddOrReplace(table_id, row_ids, col_values)
#----------------------------------------
@ -371,7 +373,7 @@ class UserActions(object):
@useraction
def UpdateRecord(self, table_id, row_id, columns):
self.BulkUpdateRecord(table_id, [row_id],
{key: [col] for key, col in columns.iteritems()})
{key: [col] for key, col in six.iteritems(columns)})
@useraction
def BulkUpdateRecord(self, table_id, row_ids, columns):
@ -380,7 +382,7 @@ class UserActions(object):
# Handle special tables, updates to which imply metadata actions.
# Check that the update is valid.
for col_id, values in columns.iteritems():
for col_id, values in six.iteritems(columns):
self._ensure_column_accepts_data(table_id, col_id, values)
# Additionally check that we are not trying to modify group-by values in a summary column
@ -413,7 +415,7 @@ class UserActions(object):
@override_action('BulkUpdateRecord', '_grist_Tables')
def _updateTableRecords(self, table_id, row_ids, col_values):
avoid_tableid_set = set(self._engine.tables.viewkeys())
avoid_tableid_set = set(self._engine.tables)
update_pairs = []
for i, rec, values in self._bulk_action_iter(table_id, row_ids, col_values):
update_pairs.append((rec, values))
@ -451,9 +453,9 @@ class UserActions(object):
if table_renames:
# Build up a dictionary mapping col_ref of each affected formula to the new formula text.
formula_updates = self._prepare_formula_renames(
{(old, None): new for (old, new) in table_renames.iteritems()})
{(old, None): new for (old, new) in six.iteritems(table_renames)})
# Add the changes to the dict of col_updates. sort for reproducible order.
for col_rec, new_formula in sorted(formula_updates.iteritems()):
for col_rec, new_formula in sorted(six.iteritems(formula_updates)):
col_updates.setdefault(col_rec, {})['formula'] = new_formula
# If a table changes to onDemand, any empty columns (formula columns with no set formula)
@ -464,7 +466,7 @@ class UserActions(object):
for col in empty_cols:
col_updates.setdefault(col, {}).update(isFormula=False, type='Text')
for col, values in col_updates.iteritems():
for col, values in six.iteritems(col_updates):
if 'type' in values:
self.doModifyColumn(col.tableId, col.colId, {'type': 'Int'})
@ -482,7 +484,7 @@ class UserActions(object):
# Internal functions are used to prevent unintended additional changes from occurring.
# Specifically, this prevents widgetOptions and displayCol from being cleared as a side
# effect of the column type change.
for col, values in col_updates.iteritems():
for col, values in six.iteritems(col_updates):
self.doModifyColumn(col.tableId, col.colId, values)
self.doBulkUpdateFromPairs('_grist_Tables_column', col_updates.items())
make_acl_updates()
@ -516,7 +518,7 @@ class UserActions(object):
# Collect all renamings that we are about to apply.
renames = {(c.parentId.tableId, c.colId): values['colId']
for c, values in col_updates.iteritems()
for c, values in six.iteritems(col_updates)
if has_diff_value(values, 'colId', c.colId)}
if renames:
@ -524,7 +526,7 @@ class UserActions(object):
formula_updates = self._prepare_formula_renames(renames)
# For any affected columns, include the formula into the update.
for col_rec, new_formula in sorted(formula_updates.iteritems()):
for col_rec, new_formula in sorted(six.iteritems(formula_updates)):
col_updates.setdefault(col_rec, {}).setdefault('formula', new_formula)
update_pairs = col_updates.items()
@ -534,7 +536,7 @@ class UserActions(object):
if col.summarySourceCol:
underlying = col_updates.get(col.summarySourceCol, {})
if not all(value == getattr(col, key) or has_value(underlying, key, value)
for key, value in values.iteritems()):
for key, value in six.iteritems(values)):
raise ValueError("Cannot modify summary group-by column '%s'" % col.colId)
make_acl_updates = acl.prepare_acl_col_renames(self._docmodel, self, renames)
@ -576,7 +578,7 @@ class UserActions(object):
def _updateACLRules(self, table_id, row_ids, col_values):
# Automatically populate aclFormulaParsed value by parsing aclFormula.
if 'aclFormula' in col_values:
col_values['aclFormulaParsed'] = map(parse_acl_formula_json, col_values['aclFormula'])
col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']]
return self.doBulkUpdateRecord(table_id, row_ids, col_values)
def _prepare_formula_renames(self, renames):
@ -605,7 +607,7 @@ class UserActions(object):
# Apply the collected patches to each affected formula, converting to unicode to apply the
# patches and back to byte string for how we maintain string values.
result = {}
for col_rec, patches in patches_map.iteritems():
for col_rec, patches in six.iteritems(patches_map):
formula = col_rec.formula.decode('utf8')
replacer = textbuilder.Replacer(textbuilder.Text(formula), patches)
result[col_rec] = replacer.get_text().encode('utf8')
@ -1066,7 +1068,7 @@ class UserActions(object):
# metadata record. We implement the former interface by forwarding to the latter.
col = self._docmodel.get_column_rec(table_id, col_id)
update_values = {k: v for k, v in col_info.iteritems() if k in _modifiable_col_fields}
update_values = {k: v for k, v in six.iteritems(col_info) if k in _modifiable_col_fields}
if '_position' in col_info:
update_values['parentPos'] = col_info['_position']
self._docmodel.update([col], **update_values)
@ -1093,7 +1095,7 @@ class UserActions(object):
old_col_info = schema.col_to_dict(self._engine.schema[table_id].columns[col_id],
include_id=False)
col_info = {k: v for k, v in col_info.iteritems() if old_col_info.get(k, v) != v}
col_info = {k: v for k, v in six.iteritems(col_info) if old_col_info.get(k, v) != v}
if not col_info:
log.info("useractions.ModifyColumn is a noop")
return
@ -1182,7 +1184,7 @@ class UserActions(object):
# Get the values from the columns and check which have changed.
all_row_ids = list(table.row_ids)
all_src_values = map(src_column.raw_get, all_row_ids)
all_src_values = [src_column.raw_get(r) for r in all_row_ids]
dst_column = table.get_column(dst_col_id)
changed_rows, changed_values = [], []
@ -1230,7 +1232,7 @@ class UserActions(object):
Add the given table with columns without creating views.
"""
# If needed, transform table_id into a valid identifier, and add a suffix to make it unique.
table_id = identifiers.pick_table_ident(table_id, avoid=self._engine.tables.viewkeys())
table_id = identifiers.pick_table_ident(table_id, avoid=six.viewkeys(self._engine.tables))
# Sanitize and de-duplicate column identifiers.
col_ids = [c['id'] for c in columns]

30
sandbox/grist/usertypes.py
View File

@ -161,7 +161,7 @@ class Text(BaseColumnType):
@classmethod
def is_right_type(cls, value):
return isinstance(value, (basestring, NoneType))
return isinstance(value, (six.string_types, NoneType))
@classmethod
def typeConvert(cls, value):
@ -203,11 +203,11 @@ class Bool(BaseColumnType):
# recognize. Everything else will result in alttext.
if not value:
return False
if isinstance(value, (float, int, long)):
if isinstance(value, (float, six.integer_types)):
return True
if isinstance(value, AltText):
value = str(value)
if isinstance(value, basestring):
if isinstance(value, six.string_types):
if value.lower() in ("false", "no", "0"):
return False
if value.lower() in ("true", "yes", "1"):
@ -235,7 +235,7 @@ class Int(BaseColumnType):
@classmethod
def is_right_type(cls, value):
return value is None or (isinstance(value, (int, long)) and not isinstance(value, bool) and
return value is None or (isinstance(value, six.integer_types) and not isinstance(value, bool) and
objtypes.is_int_short(value))
@ -252,7 +252,7 @@ class Numeric(BaseColumnType):
# TODO: Python distinguishes ints from floats, while JS only has floats. A value that can be
# interpreted as an int will upon being entered have type 'float', but after database reload
# will have type 'int'.
return isinstance(value, (float, int, long, NoneType)) and not isinstance(value, bool)
return isinstance(value, (float, six.integer_types, NoneType)) and not isinstance(value, bool)
class Date(Numeric):
@ -267,9 +267,9 @@ class Date(Numeric):
return moment.dt_to_ts(value)
elif isinstance(value, datetime.date):
return moment.date_to_ts(value)
elif isinstance(value, (float, int, long)):
elif isinstance(value, (float, six.integer_types)):
return float(value)
elif isinstance(value, basestring):
elif isinstance(value, six.string_types):
# We also accept a date in ISO format (YYYY-MM-DD), the time portion is optional and ignored
return moment.parse_iso_date(value)
else:
@ -277,7 +277,7 @@ class Date(Numeric):
@classmethod
def is_right_type(cls, value):
return isinstance(value, (float, int, long, NoneType))
return isinstance(value, (float, six.integer_types, NoneType))
@classmethod
def typeConvert(cls, value, date_format, timezone='UTC'): # pylint: disable=arguments-differ
@ -306,9 +306,9 @@ class DateTime(Date):
return moment.dt_to_ts(value, self.timezone)
elif isinstance(value, datetime.date):
return moment.date_to_ts(value, self.timezone)
elif isinstance(value, (float, int, long)):
elif isinstance(value, (float, six.integer_types)):
return float(value)
elif isinstance(value, basestring):
elif isinstance(value, six.string_types):
# We also accept a datetime in ISO format (YYYY-MM-DD[T]HH:mm:ss)
return moment.parse_iso(value, self.timezone)
else:
@ -330,7 +330,7 @@ class ChoiceList(BaseColumnType):
def do_convert(self, value):
if not value:
return None
elif isinstance(value, basestring):
elif isinstance(value, six.string_types):
# If it's a string that looks like JSON, try to parse it as such.
if value.startswith('['):
try:
@ -345,11 +345,11 @@ class ChoiceList(BaseColumnType):
@classmethod
def is_right_type(cls, value):
return value is None or (isinstance(value, (tuple, list)) and
all(isinstance(item, basestring) for item in value))
all(isinstance(item, six.string_types) for item in value))
@classmethod
def typeConvert(cls, value):
if isinstance(value, basestring) and not value.startswith('['):
if isinstance(value, six.string_types) and not value.startswith('['):
# Try to parse as CSV. If this doesn't work, we'll still try usual conversions later.
try:
tags = next(csv.reader([value]))
@ -383,7 +383,7 @@ class PositionNumber(BaseColumnType):
@classmethod
def is_right_type(cls, value):
# Same as Numeric, but does not support None.
return isinstance(value, (float, int, long)) and not isinstance(value, bool)
return isinstance(value, (float, six.integer_types)) and not isinstance(value, bool)
class ManualSortPos(PositionNumber):
@ -411,7 +411,7 @@ class Id(BaseColumnType):
@classmethod
def is_right_type(cls, value):
return (isinstance(value, (int, long)) and not isinstance(value, bool) and
return (isinstance(value, six.integer_types) and not isinstance(value, bool) and
objtypes.is_int_short(value))