gristlabs_grist-core/sandbox/grist/docmodel.py
Paul Fitzpatrick b82eec714a (core) move data engine code to core
Summary:
this moves sandbox/grist to core, and adds a requirements.txt
file for reconstructing the content of sandbox/thirdparty.

Test Plan:
existing tests pass.
Tested core functionality manually.  Tested docker build manually.

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2563
2020-07-29 08:57:25 -04:00

353 lines
15 KiB
Python

"""
This file provides convenient access to document metadata that is internal to the sandbox.
Specifically, it has handles to the metadata tables, and adds helpful formula columns to tables
which exist only in the sandbox and are not communicated to the client.
It is similar in purpose to DocModel.js on the client side.
"""
import itertools
import json
import acl
import records
import usertypes
import relabeling
import table
import moment
def _record_set(table_id, group_by, sort_by=None):
@usertypes.formulaType(usertypes.ReferenceList(table_id))
def func(rec, table):
lookup_table = table.docmodel.get_table(table_id)
return lookup_table.lookupRecords(sort_by=sort_by, **{group_by: rec.id})
return func
def _record_inverse(table_id, ref_col):
@usertypes.formulaType(usertypes.Reference(table_id))
def func(rec, table):
lookup_table = table.docmodel.get_table(table_id)
return lookup_table.lookupOne(**{ref_col: rec.id})
return func
class MetaTableExtras(object):
"""
Container class for enhancements to metadata table models. The members (formula methods) defined
for a nested class here will automatically be added as members to same-named metadata table.
"""
# pylint: disable=no-self-argument,no-member,unused-argument,not-an-iterable
class _grist_DocInfo(object):
def acl_resources(rec, table):
"""
Returns a map of ACL resources for use by acl.py. It is done in a formula so that it
automatically recomputes when anything changes in _grist_ACLResources table.
"""
# pylint: disable=no-self-use
return acl.build_resources(table.docmodel.get_table('_grist_ACLResources').lookupRecords())
@usertypes.formulaType(usertypes.Any())
def tzinfo(rec, table):
# pylint: disable=no-self-use
try:
return moment.tzinfo(rec.timezone)
except KeyError:
return moment.TZ_UTC
class _grist_Tables(object):
columns = _record_set('_grist_Tables_column', 'parentId', sort_by='parentPos')
viewSections = _record_set('_grist_Views_section', 'tableRef')
tableViews = _record_set('_grist_TableViews', 'tableRef')
summaryTables = _record_set('_grist_Tables', 'summarySourceTable')
def summaryKey(rec, table):
"""
Returns the tuple of sorted colRefs for summary columns. This uniquely identifies a summary
table among other summary tables for the same source table.
"""
# pylint: disable=not-an-iterable
return (tuple(sorted(int(c.summarySourceCol) for c in rec.columns if c.summarySourceCol))
if rec.summarySourceTable else None)
def setAutoRemove(rec, table):
"""Marks the table for removal if it's a summary table with no more view sections."""
table.docmodel.setAutoRemove(rec, rec.summarySourceTable and not rec.viewSections)
class _grist_Tables_column(object):
viewFields = _record_set('_grist_Views_section_field', 'colRef')
summaryGroupByColumns = _record_set('_grist_Tables_column', 'summarySourceCol')
usedByCols = _record_set('_grist_Tables_column', 'displayCol')
usedByFields = _record_set('_grist_Views_section_field', 'displayCol')
def tableId(rec, table):
return rec.parentId.tableId
def numDisplayColUsers(rec, table):
"""
Returns the number of cols and fields using this col as a display col
"""
return len(rec.usedByCols) + len(rec.usedByFields)
def setAutoRemove(rec, table):
"""Marks the col for removal if it's a display helper col with no more users."""
table.docmodel.setAutoRemove(rec,
rec.colId.startswith('gristHelper_Display') and rec.numDisplayColUsers == 0)
class _grist_Views(object):
viewSections = _record_set('_grist_Views_section', 'parentId')
tabBarItems = _record_set('_grist_TabBar', 'viewRef')
tableViewItems = _record_set('_grist_TableViews', 'viewRef')
primaryViewTable = _record_inverse('_grist_Tables', 'primaryViewId')
pageItems = _record_set('_grist_Pages', 'viewRef')
class _grist_Views_section(object):
fields = _record_set('_grist_Views_section_field', 'parentId', sort_by='parentPos')
class _grist_ACLRules(object):
# The set of rules that applies to this resource
@usertypes.formulaType(usertypes.ReferenceList('_grist_ACLPrincipals'))
def principalsList(rec, table):
return json.loads(rec.principals)
class _grist_ACLResources(object):
# The set of rules that applies to this resource
ruleset = _record_set('_grist_ACLRules', 'resource')
class _grist_ACLPrincipals(object):
# Memberships table maintains containment relationships between principals.
memberships = _record_set('_grist_ACLMemberships', 'parent')
# Children of a User principal are Instances. Children of a Group are Users or other Groups.
@usertypes.formulaType(usertypes.ReferenceList('_grist_ACLPrincipals'))
def children(rec, table):
return [m.child for m in rec.memberships]
@usertypes.formulaType(usertypes.ReferenceList('_grist_ACLPrincipals'))
def descendants(rec, table):
"""
Descendants through great-grandchildren. (We don't support fully recursive descendants yet,
which may be cleaner.) The max supported level is a group containing subgroups (children),
which contain users (grandchildren), which contain instances (great-grandchildren).
"""
# Include direct children.
ret = set(rec.children)
ret.add(rec)
for c1 in rec.children:
# Include grandchildren (children of each child)
ret.update(c1.children)
for c2 in c1.children:
# Include great-grandchildren (children of each grandchild).
ret.update(c2.children)
return ret
@usertypes.formulaType(usertypes.ReferenceList('_grist_ACLPrincipals'))
def allInstances(rec, table):
return sorted(r for r in rec.descendants if r.instanceId)
@usertypes.formulaType(usertypes.Text())
def name(rec, table):
return ('User:' + rec.userEmail if rec.type == 'user' else
'Group:' + rec.groupName if rec.type == 'group' else
'Inst:' + rec.instanceId if rec.type == 'instance' else '')
def enhance_model(model_class):
"""
Given a metadata model class, add all members (formula methods) to it from the same-named inner
class of MetaTableExtras. The added members are marked as private; the resulting Column objects
will have col.is_private() as true.
"""
extras_class = getattr(MetaTableExtras, model_class.__name__, None)
if not extras_class:
return
for name, member in extras_class.__dict__.iteritems():
if not name.startswith("__"):
member.__name__ = name
member.is_private = True
setattr(model_class, name, member)
# There is a single instance of DocModel per sandbox process and
# global_docmodel is a reference to it
global_docmodel = None
class DocModel(object):
"""
This class defines more convenient handles to all metadata tables. In addition, it sets
table.docmodel member for each of these tables to itself. Note that it deals with
table.UserTable objects (rather than the lower-level table.Table objects).
"""
def __init__(self, engine):
self._engine = engine
global global_docmodel # pylint: disable=global-statement
if not global_docmodel:
global_docmodel = self
# Set of records scheduled for automatic removal.
self._auto_remove_set = set()
def update_tables(self):
"""
Update the table handles we maintain to correspond to the current Engine tables.
"""
self.doc_info = self._prep_table("_grist_DocInfo")
self.tables = self._prep_table("_grist_Tables")
self.columns = self._prep_table("_grist_Tables_column")
self.table_views = self._prep_table("_grist_TableViews")
self.tab_bar = self._prep_table("_grist_TabBar")
self.views = self._prep_table("_grist_Views")
self.view_sections = self._prep_table("_grist_Views_section")
self.view_fields = self._prep_table("_grist_Views_section_field")
self.validations = self._prep_table("_grist_Validations")
self.repl_hist = self._prep_table("_grist_REPL_Hist")
self.attachments = self._prep_table("_grist_Attachments")
self.acl_rules = self._prep_table("_grist_ACLRules")
self.acl_resources = self._prep_table("_grist_ACLResources")
self.acl_principals = self._prep_table("_grist_ACLPrincipals")
self.acl_memberships = self._prep_table("_grist_ACLMemberships")
self.pages = self._prep_table("_grist_Pages")
def _prep_table(self, name):
"""
Helper that gets the table with the given name, and sets its .doc attribute to DocModel.
"""
user_table = self._engine.tables[name].user_table
user_table.docmodel = self
return user_table
def get_table(self, table_id):
return self._engine.tables[table_id].user_table
def get_table_rec(self, table_id):
"""Returns the table record for the given table name, or raises ValueError."""
table_rec = self.tables.lookupOne(tableId=table_id)
if not table_rec:
raise ValueError("No such table: %s" % table_id)
return table_rec
def get_column_rec(self, table_id, col_id):
"""Returns the column record for the given table and column names, or raises ValueError."""
col_rec = self.columns.lookupOne(tableId=table_id, colId=col_id)
if not col_rec:
raise ValueError("No such column: %s.%s" % (table_id, col_id))
return col_rec
def setAutoRemove(self, record, yes_or_no):
"""
Marks a record for automatic removal. To use, create a formula in your table, e.g.
'setAutoRemove', which calls `table.docmodel.setAutoRemove(boolean_value)`. Whenever it gets
reevaluated and the boolean_value is true, the record will be automatically removed.
For now, it is only usable in metadata tables, although we could extend to user tables.
"""
if yes_or_no:
self._auto_remove_set.add(record)
else:
self._auto_remove_set.discard(record)
def apply_auto_removes(self):
"""
Remove the records marked for removal.
"""
# Sort to make sure removals are done in deterministic order.
gone_records = sorted(self._auto_remove_set)
self._auto_remove_set.clear()
self.remove(gone_records)
return bool(gone_records)
def remove(self, records):
"""
Removes all records in the given iterable of Records.
"""
for table_id, group in itertools.groupby(records, lambda r: r._table.table_id):
self._engine.user_actions.BulkRemoveRecord(table_id, [int(r) for r in group])
def update(self, records, **col_values):
"""
Updates all records in the given list of Records or a RecordSet; col_values maps column ids to
values. The values may either be a list of the length len(records), or a non-list value that
will be used for all records.
"""
record_list = list(records)
if not record_list:
return
table_id = record_list[0]._table.table_id
# Make sure these are all records from the same table.
assert all(r._table.table_id == table_id for r in record_list)
row_ids = [int(r) for r in record_list]
values = _unify_col_values(col_values, len(record_list))
self._engine.user_actions.BulkUpdateRecord(table_id, row_ids, values)
def add(self, record_set_or_table, **col_values):
"""
Add new records for the given table; col_values maps column ids to values. Values may either
be lists (all of the same length), or non-list values that will be used for all added records.
Either a UserTable or a RecordSet may used as the first argument. If it is a RecordSet created
with lookupRecords, it may set additional col_values.
Returns a list of inserted records.
"""
assert isinstance(record_set_or_table, (records.RecordSet, table.UserTable))
count = _get_col_values_count(col_values)
values = _unify_col_values(col_values, count)
if isinstance(record_set_or_table, records.RecordSet):
table_obj = record_set_or_table._table
group_by = record_set_or_table._group_by
if group_by:
values.update((k, [v] * count) for k, v in group_by.iteritems() if k not in values)
else:
table_obj = record_set_or_table.table
row_ids = self._engine.user_actions.BulkAddRecord(table_obj.table_id, [None] * count, values)
return [table_obj.Record(table_obj, r, None) for r in row_ids]
def insert(self, record_set, position, **col_values):
"""
Add new records using col_values, inserting them into record_set according to position.
This may only be used when record_set is sorted by a field of type PositionNumber; in
particular it must be the result of lookupRecords() with 'sort_by' parameter.
Position may be numeric (to compare to other sort_by values), or None to insert at the end.
Returns a list of inserted records.
"""
assert isinstance(record_set, records.RecordSet), \
"docmodel.insert() may only be used on a RecordSet, not %s" % type(record_set)
sort_by = getattr(record_set, '_sort_by', None)
assert sort_by, \
"docmodel.insert() may only be used on a sorted RecordSet"
column = record_set._table.get_column(sort_by)
assert isinstance(column.type_obj, usertypes.PositionNumber), \
"docmodel.insert() may only be used on a RecordSet sorted by PositionNumber type column"
col_values[sort_by] = float('inf') if position is None else position
return self.add(record_set, **col_values)
def insert_after(self, record_set, position, **col_values):
"""
Same as insert, but when position is equal to the position of an existing record, inserts
after that record; and when position is None, inserts at the beginning.
"""
# We can reuse insert() by just using the next float for position. As long as positions of
# existing records are different, that would necessarily place the new records correctly.
pos = float('-inf') if position is None else relabeling.nextfloat(position)
return self.insert(record_set, pos, **col_values)
def _unify_col_values(col_values, count):
"""
Helper that converts a dict mapping keys to values or lists of values to all lists. Non-list
values get turned into lists by repeating them count times.
"""
assert all(len(v) == count for v in col_values.itervalues() if isinstance(v, list))
return {k: (v if isinstance(v, list) else [v] * count)
for k, v in col_values.iteritems()}
def _get_col_values_count(col_values):
"""
Helper that returns the length of the first list in among the values of col_values. If none of
the values is a list, returns 1.
"""
first_list = next((v for v in col_values.itervalues() if isinstance(v, list)), None)
return len(first_list) if first_list is not None else 1