2021-05-12 14:34:49 +00:00
|
|
|
import json
|
2020-07-27 18:57:36 +00:00
|
|
|
import types
|
|
|
|
from collections import namedtuple
|
|
|
|
|
|
|
|
import depend
|
|
|
|
import objtypes
|
|
|
|
import usertypes
|
|
|
|
import relabeling
|
|
|
|
import relation
|
|
|
|
import moment
|
|
|
|
import logger
|
|
|
|
from sortedcontainers import SortedListWithKey
|
|
|
|
|
|
|
|
log = logger.Logger(__name__, logger.INFO)
|
|
|
|
|
|
|
|
MANUAL_SORT = 'manualSort'
|
|
|
|
MANUAL_SORT_COL_INFO = {
|
|
|
|
'id': MANUAL_SORT,
|
|
|
|
'type': 'ManualSortPos',
|
|
|
|
'formula': '',
|
|
|
|
'isFormula': False
|
|
|
|
}
|
|
|
|
MANUAL_SORT_DEFAULT = 2147483647.0
|
|
|
|
|
|
|
|
SPECIAL_COL_IDS = {'id', MANUAL_SORT}
|
|
|
|
|
|
|
|
def is_user_column(col_id):
|
|
|
|
"""
|
|
|
|
Returns whether the col_id is of a user column (as opposed to special columns that can't be used
|
|
|
|
for user data).
|
|
|
|
"""
|
|
|
|
return col_id not in SPECIAL_COL_IDS and not col_id.startswith('#')
|
|
|
|
|
|
|
|
def is_visible_column(col_id):
|
|
|
|
"""
|
|
|
|
Returns whether this is an id of a column that's intended to be shown to the user.
|
|
|
|
"""
|
|
|
|
return is_user_column(col_id) and not col_id.startswith('gristHelper_')
|
|
|
|
|
|
|
|
def is_virtual_column(col_id):
|
|
|
|
"""
|
|
|
|
Returns whether col_id is of a special column that does not get communicated outside of the
|
|
|
|
sandbox. Lookup maps are an example.
|
|
|
|
"""
|
|
|
|
return col_id.startswith('#')
|
|
|
|
|
|
|
|
def is_validation_column_name(name):
|
|
|
|
return name.startswith("validation___")
|
|
|
|
|
|
|
|
ColInfo = namedtuple('ColInfo', ('type_obj', 'is_formula', 'method'))
|
|
|
|
|
|
|
|
def get_col_info(col_model, default_func=None):
|
|
|
|
if isinstance(col_model, types.FunctionType):
|
|
|
|
type_obj = getattr(col_model, 'grist_type', usertypes.Any())
|
|
|
|
return ColInfo(type_obj, is_formula=True, method=col_model)
|
|
|
|
else:
|
|
|
|
return ColInfo(col_model, is_formula=False, method=col_model.default_func or default_func)
|
|
|
|
|
|
|
|
|
|
|
|
class BaseColumn(object):
|
|
|
|
"""
|
|
|
|
BaseColumn holds a column of data, whether raw or computed.
|
|
|
|
"""
|
|
|
|
def __init__(self, table, col_id, col_info):
|
|
|
|
self.type_obj = col_info.type_obj
|
|
|
|
self._data = []
|
|
|
|
self.col_id = col_id
|
|
|
|
self.table_id = table.table_id
|
|
|
|
self.node = depend.Node(self.table_id, col_id)
|
|
|
|
self._is_formula = col_info.is_formula
|
|
|
|
self._is_private = bool(col_info.method) and getattr(col_info.method, 'is_private', False)
|
|
|
|
self.method = col_info.method
|
|
|
|
|
|
|
|
# Always initialize to include the special empty record at index 0.
|
|
|
|
self.growto(1)
|
|
|
|
|
|
|
|
def update_method(self, method):
|
|
|
|
"""
|
|
|
|
After rebuilding user code, we reuse existing column objects, but need to replace their
|
|
|
|
'method' function. The method may refer to variables in the generated "usercode" module, and
|
|
|
|
it's important that all such references are to the rebuilt "usercode" module.
|
|
|
|
"""
|
|
|
|
self.method = method
|
|
|
|
|
|
|
|
def is_formula(self):
|
|
|
|
"""
|
|
|
|
Whether this is a formula column. Note that a non-formula column may have an associated
|
|
|
|
method, which is used to fill in defaults when a record is added.
|
|
|
|
"""
|
|
|
|
return self._is_formula
|
|
|
|
|
|
|
|
def is_private(self):
|
|
|
|
"""
|
|
|
|
Returns whether this method is private to the sandbox. If so, changes to this column do not
|
|
|
|
get communicated to outside the sandbox via actions.
|
|
|
|
"""
|
|
|
|
return self._is_private
|
|
|
|
|
|
|
|
def has_formula(self):
|
|
|
|
"""
|
|
|
|
has_formula is true if formula is set, whether or not this is a formula column.
|
|
|
|
"""
|
|
|
|
return self.method is not None
|
|
|
|
|
|
|
|
def clear(self):
|
|
|
|
self._data = []
|
|
|
|
self.growto(1) # Always include the special empty record at index 0.
|
|
|
|
|
|
|
|
def destroy(self):
|
|
|
|
"""
|
|
|
|
Called when the column is deleted.
|
|
|
|
"""
|
|
|
|
del self._data[:]
|
|
|
|
|
|
|
|
def growto(self, size):
|
|
|
|
if len(self._data) < size:
|
|
|
|
self._data.extend([self.getdefault()] * (size - len(self._data)))
|
|
|
|
|
|
|
|
def size(self):
|
|
|
|
return len(self._data)
|
|
|
|
|
|
|
|
def set(self, row_id, value):
|
|
|
|
"""
|
|
|
|
Sets the value of this column for the given row_id. Value should be as returned by convert(),
|
|
|
|
i.e. of the right type, or alttext, or error (but should NOT be random wrong types).
|
|
|
|
"""
|
|
|
|
try:
|
|
|
|
self._data[row_id] = value
|
|
|
|
except IndexError:
|
|
|
|
self.growto(row_id + 1)
|
|
|
|
self._data[row_id] = value
|
|
|
|
|
|
|
|
def unset(self, row_id):
|
|
|
|
"""
|
|
|
|
Sets the value for the given row_id to the default value.
|
|
|
|
"""
|
|
|
|
self.set(row_id, self.getdefault())
|
|
|
|
|
|
|
|
def get_cell_value(self, row_id):
|
|
|
|
"""
|
|
|
|
Returns the "rich" value for the given row_id, i.e. the value that would be seen by formulas.
|
|
|
|
E.g. for ReferenceColumns it'll be the referred-to Record object. For cells containing
|
|
|
|
alttext, this will be an AltText object. For RaisedException objects that represent a thrown
|
|
|
|
error, this will re-raise that error.
|
|
|
|
"""
|
|
|
|
raw = self.raw_get(row_id)
|
|
|
|
if isinstance(raw, objtypes.RaisedException):
|
|
|
|
raise raw.error
|
|
|
|
if self.type_obj.is_right_type(raw):
|
|
|
|
return self._make_rich_value(raw)
|
|
|
|
return usertypes.AltText(str(raw), self.type_obj.typename())
|
|
|
|
|
|
|
|
def _make_rich_value(self, typed_value):
|
|
|
|
"""
|
|
|
|
Called by get_cell_value() with a value of the right type for this column. Should be
|
|
|
|
implemented by derived classes to produce a "rich" version of the value.
|
|
|
|
"""
|
|
|
|
# pylint: disable=no-self-use
|
|
|
|
return typed_value
|
|
|
|
|
|
|
|
def raw_get(self, row_id):
|
|
|
|
"""
|
|
|
|
Returns the value stored for the given row_id. This may be an error or alttext, and it does
|
|
|
|
not convert to a richer object.
|
|
|
|
"""
|
|
|
|
try:
|
|
|
|
return self._data[row_id]
|
|
|
|
except IndexError:
|
|
|
|
return self.getdefault()
|
|
|
|
|
|
|
|
def safe_get(self, row_id):
|
|
|
|
"""
|
|
|
|
Returns a value of the right type, or the default value if the stored value had a wrong type.
|
|
|
|
"""
|
|
|
|
raw = self.raw_get(row_id)
|
|
|
|
return raw if self.type_obj.is_right_type(raw) else self.getdefault()
|
|
|
|
|
|
|
|
def getdefault(self):
|
|
|
|
"""
|
|
|
|
Returns the default value for this column. This is a static default; the implementation of
|
|
|
|
"default formula" logic is separate.
|
|
|
|
"""
|
|
|
|
return self.type_obj.default
|
|
|
|
|
|
|
|
def sample_value(self):
|
|
|
|
"""
|
|
|
|
Returns a sample value for this column, used for auto-completions. E.g. for a date, this
|
|
|
|
returns an actual datetime object rather than None (only its attributes should matter).
|
|
|
|
"""
|
|
|
|
return self.type_obj.default
|
|
|
|
|
|
|
|
def copy_from_column(self, other_column):
|
|
|
|
"""
|
|
|
|
Replace this column's data entirely with data from another column of the same exact type.
|
|
|
|
"""
|
|
|
|
self._data[:] = other_column._data
|
|
|
|
|
|
|
|
def convert(self, value_to_convert):
|
|
|
|
"""
|
|
|
|
Converts a value of any type to this column's type, returning either the converted value (for
|
|
|
|
which is_right_type is true), or an alttext string, or an error object.
|
|
|
|
"""
|
|
|
|
return self.type_obj.convert(value_to_convert)
|
|
|
|
|
2020-12-15 19:37:55 +00:00
|
|
|
def prepare_new_values(self, values, ignore_data=False, action_summary=None):
|
2020-07-27 18:57:36 +00:00
|
|
|
"""
|
|
|
|
This allows us to modify values and also produce adjustments to existing records. This
|
|
|
|
currently is only used by PositionColumn. Returns two lists: new_values, and
|
|
|
|
[(row_id, new_value)] list of adjustments to existing records.
|
|
|
|
If ignore_data is True, makes adjustments without regard to the existing data; this is used
|
|
|
|
for processing ReplaceTableData actions.
|
|
|
|
"""
|
|
|
|
# pylint: disable=no-self-use, unused-argument
|
|
|
|
return values, []
|
|
|
|
|
|
|
|
|
|
|
|
class DataColumn(BaseColumn):
|
|
|
|
"""
|
|
|
|
DataColumn describes a column of raw data, and holds it.
|
|
|
|
"""
|
|
|
|
pass
|
|
|
|
|
|
|
|
class BoolColumn(BaseColumn):
|
|
|
|
def set(self, row_id, value):
|
|
|
|
# When 1 or 1.0 is loaded, we should see it as True, and similarly 0 as False. This is similar
|
|
|
|
# to how, after loading a number into a DateColumn, we should see a date, except we adjust
|
|
|
|
# booleans at set() time.
|
|
|
|
bool_value = True if value == 1 else (False if value == 0 else value)
|
|
|
|
super(BoolColumn, self).set(row_id, bool_value)
|
|
|
|
|
|
|
|
class NumericColumn(BaseColumn):
|
|
|
|
def set(self, row_id, value):
|
|
|
|
# Make sure any integers are treated as floats to avoid truncation.
|
|
|
|
# Uses `type(value) == int` rather than `isintance(value, int)` to specifically target
|
|
|
|
# ints and not bools (which are singleton instances the class int in python). But
|
|
|
|
# perhaps something should be done about bools also?
|
|
|
|
# pylint: disable=unidiomatic-typecheck
|
|
|
|
super(NumericColumn, self).set(row_id, float(value) if type(value) == int else value)
|
|
|
|
|
|
|
|
_sample_date = moment.ts_to_date(0)
|
|
|
|
_sample_datetime = moment.ts_to_dt(0, None, moment.TZ_UTC)
|
|
|
|
|
2020-12-01 21:53:33 +00:00
|
|
|
class DateColumn(NumericColumn):
|
2020-07-27 18:57:36 +00:00
|
|
|
"""
|
|
|
|
DateColumn contains numerical timestamps represented as seconds since epoch, in type float,
|
|
|
|
to midnight of specific UTC dates. Accessing them yields date objects.
|
|
|
|
"""
|
|
|
|
def _make_rich_value(self, typed_value):
|
|
|
|
return typed_value and moment.ts_to_date(typed_value)
|
|
|
|
|
|
|
|
def sample_value(self):
|
|
|
|
return _sample_date
|
|
|
|
|
2020-12-01 21:53:33 +00:00
|
|
|
class DateTimeColumn(NumericColumn):
|
2020-07-27 18:57:36 +00:00
|
|
|
"""
|
|
|
|
DateTimeColumn contains numerical timestamps represented as seconds since epoch, in type float,
|
|
|
|
and a timestamp associated with the column. Accessing them yields datetime objects.
|
|
|
|
"""
|
|
|
|
def __init__(self, table, col_id, col_info):
|
|
|
|
super(DateTimeColumn, self).__init__(table, col_id, col_info)
|
|
|
|
self._timezone = col_info.type_obj.timezone
|
|
|
|
|
|
|
|
def _make_rich_value(self, typed_value):
|
|
|
|
return typed_value and moment.ts_to_dt(typed_value, self._timezone)
|
|
|
|
|
|
|
|
def sample_value(self):
|
|
|
|
return _sample_datetime
|
|
|
|
|
2020-12-01 21:53:33 +00:00
|
|
|
class PositionColumn(NumericColumn):
|
2020-07-27 18:57:36 +00:00
|
|
|
def __init__(self, table, col_id, col_info):
|
|
|
|
super(PositionColumn, self).__init__(table, col_id, col_info)
|
|
|
|
# This is a list of row_ids, ordered by the position.
|
|
|
|
self._sorted_rows = SortedListWithKey(key=self.raw_get)
|
|
|
|
|
|
|
|
def set(self, row_id, value):
|
|
|
|
self._sorted_rows.discard(row_id)
|
|
|
|
super(PositionColumn, self).set(row_id, value)
|
|
|
|
if value != self.getdefault():
|
|
|
|
self._sorted_rows.add(row_id)
|
|
|
|
|
|
|
|
def copy_from_column(self, other_column):
|
|
|
|
super(PositionColumn, self).copy_from_column(other_column)
|
|
|
|
self._sorted_rows = SortedListWithKey(other_column._sorted_rows[:], key=self.raw_get)
|
|
|
|
|
2020-12-15 19:37:55 +00:00
|
|
|
def prepare_new_values(self, values, ignore_data=False, action_summary=None):
|
2020-07-27 18:57:36 +00:00
|
|
|
# This does the work of adjusting positions and relabeling existing rows with new position
|
|
|
|
# (without changing sort order) to make space for the new positions. Note that this is also
|
|
|
|
# used for updating a position for an existing row: we'll find a new value for it; later when
|
|
|
|
# this value is set, the old position will be removed and the new one added.
|
|
|
|
if ignore_data:
|
|
|
|
rows = SortedListWithKey([], key=self.raw_get)
|
|
|
|
else:
|
|
|
|
rows = self._sorted_rows
|
|
|
|
adjustments, new_values = relabeling.prepare_inserts(rows, values)
|
|
|
|
return new_values, [(self._sorted_rows[i], pos) for (i, pos) in adjustments]
|
|
|
|
|
|
|
|
|
2021-05-12 14:34:49 +00:00
|
|
|
class ChoiceListColumn(BaseColumn):
|
|
|
|
"""
|
|
|
|
ChoiceListColumn's default value is None, but is presented to formulas as the empty list.
|
|
|
|
"""
|
|
|
|
def set(self, row_id, value):
|
|
|
|
# When a JSON string is loaded, set it to a tuple parsed from it. When a list is loaded,
|
|
|
|
# convert to a tuple to keep values immutable.
|
|
|
|
if isinstance(value, basestring) and value.startswith('['):
|
|
|
|
try:
|
|
|
|
value = tuple(json.loads(value))
|
|
|
|
except Exception:
|
|
|
|
pass
|
|
|
|
elif isinstance(value, list):
|
|
|
|
value = tuple(value)
|
|
|
|
super(ChoiceListColumn, self).set(row_id, value)
|
|
|
|
|
|
|
|
def _make_rich_value(self, typed_value):
|
|
|
|
return () if typed_value is None else typed_value
|
|
|
|
|
|
|
|
|
2020-07-27 18:57:36 +00:00
|
|
|
class BaseReferenceColumn(BaseColumn):
|
|
|
|
"""
|
|
|
|
Base class for ReferenceColumn and ReferenceListColumn.
|
|
|
|
"""
|
|
|
|
def __init__(self, table, col_id, col_info):
|
|
|
|
super(BaseReferenceColumn, self).__init__(table, col_id, col_info)
|
|
|
|
# We can assume that all tables have been instantiated, but not all initialized.
|
|
|
|
target_table_id = self.type_obj.table_id
|
|
|
|
self._target_table = table._engine.tables.get(target_table_id, None)
|
|
|
|
self._relation = relation.ReferenceRelation(table.table_id, target_table_id, col_id)
|
|
|
|
# Note that we need to remove these back-references when the column is removed.
|
|
|
|
if self._target_table:
|
|
|
|
self._target_table._back_references.add(self)
|
|
|
|
|
|
|
|
def destroy(self):
|
|
|
|
# Destroy the column and remove the back-reference we created in the constructor.
|
|
|
|
super(BaseReferenceColumn, self).destroy()
|
|
|
|
if self._target_table:
|
|
|
|
self._target_table._back_references.remove(self)
|
|
|
|
|
|
|
|
def _update_references(self, row_id, old_value, new_value):
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
|
|
|
def set(self, row_id, value):
|
|
|
|
old = self.safe_get(row_id)
|
|
|
|
super(BaseReferenceColumn, self).set(row_id, value)
|
|
|
|
new = self.safe_get(row_id)
|
|
|
|
self._update_references(row_id, old, new)
|
|
|
|
|
|
|
|
def copy_from_column(self, other_column):
|
|
|
|
super(BaseReferenceColumn, self).copy_from_column(other_column)
|
|
|
|
self._relation.clear()
|
|
|
|
# This is hacky: we should have an interface to iterate through values of a column. (As it is,
|
|
|
|
# self._data may include values for non-existent rows; it works here because those values are
|
|
|
|
# falsy, which makes them ignored by self._update_references).
|
|
|
|
for row_id, value in enumerate(self._data):
|
2020-11-25 07:35:44 +00:00
|
|
|
if self.type_obj.is_right_type(value):
|
2020-07-27 18:57:36 +00:00
|
|
|
self._update_references(row_id, None, value)
|
|
|
|
|
|
|
|
def sample_value(self):
|
|
|
|
return self._target_table.sample_record
|
|
|
|
|
|
|
|
|
|
|
|
class ReferenceColumn(BaseReferenceColumn):
|
|
|
|
"""
|
|
|
|
ReferenceColumn contains IDs of rows in another table. Accessing them yields the records in the
|
|
|
|
other table.
|
|
|
|
"""
|
|
|
|
def _make_rich_value(self, typed_value):
|
|
|
|
# If we refer to an invalid table, return integers rather than fail completely.
|
|
|
|
if not self._target_table:
|
|
|
|
return typed_value
|
|
|
|
# For a Reference, values must either refer to an existing record, or be 0. In all tables,
|
|
|
|
# the 0 index will contain the all-defaults record.
|
|
|
|
return self._target_table.Record(self._target_table, typed_value, self._relation)
|
|
|
|
|
|
|
|
def _update_references(self, row_id, old_value, new_value):
|
|
|
|
if old_value:
|
|
|
|
self._relation.remove_reference(row_id, old_value)
|
|
|
|
if new_value:
|
|
|
|
self._relation.add_reference(row_id, new_value)
|
|
|
|
|
2020-12-15 19:37:55 +00:00
|
|
|
def prepare_new_values(self, values, ignore_data=False, action_summary=None):
|
|
|
|
if action_summary and values:
|
|
|
|
values = action_summary.translate_new_row_ids(self._target_table.table_id, values)
|
|
|
|
return values, []
|
|
|
|
|
2020-07-27 18:57:36 +00:00
|
|
|
|
|
|
|
class ReferenceListColumn(BaseReferenceColumn):
|
|
|
|
"""
|
|
|
|
ReferenceListColumn maintains for each row a list of references (row IDs) into another table.
|
|
|
|
Accessing them yields RecordSets.
|
|
|
|
"""
|
|
|
|
def _update_references(self, row_id, old_list, new_list):
|
|
|
|
for old_value in old_list or ():
|
|
|
|
self._relation.remove_reference(row_id, old_value)
|
|
|
|
for new_value in new_list or ():
|
|
|
|
self._relation.add_reference(row_id, new_value)
|
|
|
|
|
|
|
|
def _make_rich_value(self, typed_value):
|
|
|
|
if typed_value is None:
|
|
|
|
typed_value = []
|
|
|
|
# If we refer to an invalid table, return integers rather than fail completely.
|
|
|
|
if not self._target_table:
|
|
|
|
return typed_value
|
|
|
|
return self._target_table.RecordSet(self._target_table, typed_value, self._relation)
|
|
|
|
|
|
|
|
|
|
|
|
# Set up the relationship between usertypes objects and column objects.
|
|
|
|
usertypes.BaseColumnType.ColType = DataColumn
|
|
|
|
usertypes.Reference.ColType = ReferenceColumn
|
|
|
|
usertypes.ReferenceList.ColType = ReferenceListColumn
|
2021-05-12 14:34:49 +00:00
|
|
|
usertypes.ChoiceList.ColType = ChoiceListColumn
|
2020-07-27 18:57:36 +00:00
|
|
|
usertypes.DateTime.ColType = DateTimeColumn
|
|
|
|
usertypes.Date.ColType = DateColumn
|
|
|
|
usertypes.PositionNumber.ColType = PositionColumn
|
|
|
|
usertypes.Bool.ColType = BoolColumn
|
|
|
|
usertypes.Numeric.ColType = NumericColumn
|
|
|
|
|
|
|
|
def create_column(table, col_id, col_info):
|
|
|
|
return col_info.type_obj.ColType(table, col_id, col_info)
|