mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
b58929f095
Summary: Due to a mishap, two distinct migrations with the same migration number were introduced into Grist. This diff reconciles them as best we can, by adding another migration to make sure both desired changes have run (and running them if not). Test Plan: updated a test; checked manually that documents with different 38 migrations are handled as expected. Reviewers: georgegevoian, jarek Reviewed By: georgegevoian, jarek Differential Revision: https://phab.getgrist.com/D3895
1237 lines
48 KiB
Python
1237 lines
48 KiB
Python
import json
|
|
import re
|
|
from collections import defaultdict
|
|
|
|
import six
|
|
from six.moves import xrange
|
|
|
|
import actions
|
|
import identifiers
|
|
import schema
|
|
import summary
|
|
import table_data_set
|
|
import logger
|
|
from column import is_visible_column
|
|
|
|
log = logger.Logger(__name__, logger.INFO)
|
|
|
|
# PHILOSOPHY OF MIGRATIONS.
|
|
#
|
|
# We should probably never remove, modify, or rename metadata tables or columns.
|
|
# Instead, we should only add.
|
|
#
|
|
# We can mark old columns/tables as deprecated, which should be ignored except to prevent us from
|
|
# adding same-named entities in the future.
|
|
#
|
|
# If we change the meaning of a column, we have to create a new column with a new name.
|
|
#
|
|
# This should make it at least barely possible to share documents by people who are not all on the
|
|
# same Grist version (even so, it will require more work). It should also make it somewhat safe to
|
|
# upgrade and then open the document with a previous version.
|
|
#
|
|
# After each migration you probably should run these commands:
|
|
# ./test/upgradeDocument public_samples/*.grist
|
|
# UPDATE_REGRESSION_DATA=1 GREP_TESTS=DocRegressionTests ./test/testrun.sh server
|
|
# ./test/upgradeDocument core/test/fixtures/docs/Hello.grist
|
|
|
|
all_migrations = {}
|
|
|
|
def noop_migration(_all_tables):
|
|
return []
|
|
|
|
# Each migration function includes a .need_all_tables attribute. See migration() decorator.
|
|
noop_migration.need_all_tables = False
|
|
|
|
|
|
def create_migrations(all_tables, metadata_only=False):
|
|
"""
|
|
Creates and returns a list of DocActions needed to bring this document to
|
|
schema.SCHEMA_VERSION.
|
|
all_tables: all tables or just the metadata tables (those named with _grist_ prefix) as a
|
|
dictionary mapping table name to TableData.
|
|
metadata_only: should be set if only metadata tables are passed in. If ALL tables are
|
|
required to process migrations, this method will raise a "need all tables..." exception.
|
|
"""
|
|
try:
|
|
doc_version = all_tables['_grist_DocInfo'].columns["schemaVersion"][0]
|
|
except Exception:
|
|
doc_version = 0
|
|
|
|
# We create a TableDataSet, and populate it with the subset of the current schema that matches
|
|
# all_tables. For missing items, we make up tables and incomplete columns, which should be OK
|
|
# since we would not be adding new records to deprecated columns.
|
|
# Note that this approach makes it NOT OK to change column types.
|
|
tdset = table_data_set.TableDataSet()
|
|
|
|
# For each table in the provided metadata tables, create an AddTable action.
|
|
user_schema = schema.build_schema(all_tables['_grist_Tables'],
|
|
all_tables['_grist_Tables_column'],
|
|
include_builtin=False)
|
|
for t in six.itervalues(user_schema):
|
|
tdset.apply_doc_action(actions.AddTable(t.tableId, schema.cols_to_dict_list(t.columns)))
|
|
|
|
# For each old table/column, construct an AddTable action using the current schema.
|
|
new_schema = {a.table_id: a for a in schema.schema_create_actions()}
|
|
for table_id, data in sorted(six.iteritems(all_tables)):
|
|
# User tables should already be in tdset; the rest must be metadata tables.
|
|
# (If metadata_only is true, there is simply nothing to skip here.)
|
|
if table_id not in tdset.all_tables:
|
|
new_col_info = {}
|
|
if table_id in new_schema:
|
|
new_col_info = {c['id']: c for c in new_schema[table_id].columns}
|
|
# Use an incomplete default for unknown (i.e. deprecated) columns; some uses of the column
|
|
# would be invalid, such as adding a new record with missing values.
|
|
col_info = sorted([new_col_info.get(col_id, {'id': col_id}) for col_id in data.columns],
|
|
key=lambda c: list(six.iteritems(c)))
|
|
tdset.apply_doc_action(actions.AddTable(table_id, col_info))
|
|
|
|
# And load in the original data, interpreting the TableData object as BulkAddRecord action.
|
|
tdset.apply_doc_action(actions.BulkAddRecord(*data))
|
|
|
|
migration_actions = []
|
|
for version in xrange(doc_version + 1, schema.SCHEMA_VERSION + 1):
|
|
migration_func = all_migrations.get(version, noop_migration)
|
|
if migration_func.need_all_tables and metadata_only:
|
|
raise Exception("need all tables for migration to %s" % version)
|
|
migration_actions.extend(all_migrations.get(version, noop_migration)(tdset))
|
|
|
|
# Note that if we are downgrading versions (i.e. doc_version is higher), then the following is
|
|
# the only action we include into the migration.
|
|
migration_actions.append(actions.UpdateRecord('_grist_DocInfo', 1, {
|
|
'schemaVersion': schema.SCHEMA_VERSION
|
|
}))
|
|
return migration_actions
|
|
|
|
def get_last_migration_version():
|
|
"""
|
|
Returns the last schema version number for which we have a migration defined.
|
|
"""
|
|
return max(all_migrations)
|
|
|
|
def migration(schema_version, need_all_tables=False):
|
|
"""
|
|
Decorator for migrations that associates the decorated migration function with the given
|
|
schema_version. This decorated function will be run to migrate forward to schema_version.
|
|
|
|
Migrations are first attempted with only metadata tables, but if any required migration function
|
|
is marked with need_all_tables=True, then the migration will be retried with all tables.
|
|
|
|
NOTE: new migrations should NOT set need_all_tables=True; it would require more work to process
|
|
very large documents safely (including those containing on-demand tables).
|
|
"""
|
|
def add_migration(migration_func):
|
|
migration_func.need_all_tables = need_all_tables
|
|
all_migrations[schema_version] = migration_func
|
|
return migration_func
|
|
return add_migration
|
|
|
|
# A little shorthand to make AddColumn actions more concise.
|
|
def add_column(table_id, col_id, col_type, *args, **kwargs):
|
|
return actions.AddColumn(table_id, col_id,
|
|
schema.make_column(col_id, col_type, *args, **kwargs))
|
|
|
|
# Another shorthand to only add a column if it isn't already there.
|
|
def maybe_add_column(tdset, table_id, col_id, col_type, *args, **kwargs):
|
|
if col_id not in tdset.all_tables[table_id].columns:
|
|
return add_column(table_id, col_id, col_type, *args, **kwargs)
|
|
return None
|
|
|
|
# Returns the next unused row id for the records of the table given by table_id.
|
|
def next_id(tdset, table_id):
|
|
row_ids = tdset.all_tables[table_id].row_ids
|
|
return max(row_ids) + 1 if row_ids else 1
|
|
|
|
# Parses a json string, but returns an empty object for invalid json.
|
|
def safe_parse(json_str):
|
|
try:
|
|
return json.loads(json_str)
|
|
except ValueError:
|
|
return {}
|
|
|
|
@migration(schema_version=1)
|
|
def migration1(tdset):
|
|
"""
|
|
Add TabItems table, and populate based on existing sections.
|
|
"""
|
|
doc_actions = []
|
|
|
|
# The very first migration is extra-lax, and creates some tables that are missing in some test
|
|
# docs. That's only because we did not distinguish schema version before migrations were
|
|
# implemented. Other migrations should not need such conditionals.
|
|
if '_grist_Attachments' not in tdset.all_tables:
|
|
doc_actions.append(actions.AddTable("_grist_Attachments", [
|
|
schema.make_column("fileIdent", "Text"),
|
|
schema.make_column("fileName", "Text"),
|
|
schema.make_column("fileType", "Text"),
|
|
schema.make_column("fileSize", "Int"),
|
|
schema.make_column("timeUploaded", "DateTime")
|
|
]))
|
|
|
|
if '_grist_TabItems' not in tdset.all_tables:
|
|
doc_actions.append(actions.AddTable("_grist_TabItems", [
|
|
schema.make_column("tableRef", "Ref:_grist_Tables"),
|
|
schema.make_column("viewRef", "Ref:_grist_Views"),
|
|
]))
|
|
|
|
if 'schemaVersion' not in tdset.all_tables['_grist_DocInfo'].columns:
|
|
doc_actions.append(add_column('_grist_DocInfo', 'schemaVersion', 'Int'))
|
|
|
|
doc_actions.extend([
|
|
add_column('_grist_Attachments', 'imageHeight', 'Int'),
|
|
add_column('_grist_Attachments', 'imageWidth', 'Int'),
|
|
])
|
|
|
|
view_sections = actions.transpose_bulk_action(tdset.all_tables['_grist_Views_section'])
|
|
rows = sorted({(s.tableRef, s.parentId) for s in view_sections})
|
|
if rows:
|
|
values = {'tableRef': [r[0] for r in rows],
|
|
'viewRef': [r[1] for r in rows]}
|
|
row_ids = list(xrange(1, len(rows) + 1))
|
|
doc_actions.append(actions.ReplaceTableData('_grist_TabItems', row_ids, values))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=2)
|
|
def migration2(tdset):
|
|
"""
|
|
Add TableViews table, and populate based on existing sections.
|
|
Add TabBar table, and populate based on existing views.
|
|
Add PrimaryViewId to Tables and populated using relatedViews
|
|
"""
|
|
# Maps tableRef to viewRef
|
|
primary_views = {}
|
|
|
|
# Associate each view with a single table; this dict includes primary views.
|
|
views_to_table = {}
|
|
|
|
# For each table, find a view to serve as the primary view.
|
|
view_sections = actions.transpose_bulk_action(tdset.all_tables['_grist_Views_section'])
|
|
for s in view_sections:
|
|
if s.tableRef not in primary_views and s.parentKey == "record":
|
|
# The view containing this section is a good candidate for primary view.
|
|
primary_views[s.tableRef] = s.parentId
|
|
|
|
if s.parentId not in views_to_table:
|
|
# The first time we see a (view, table) combination, associate the view with that table.
|
|
views_to_table[s.parentId] = s.tableRef
|
|
|
|
def create_primary_views_action(primary_views):
|
|
row_ids = sorted(primary_views.keys())
|
|
values = {'primaryViewId': [primary_views[r] for r in row_ids]}
|
|
return actions.BulkUpdateRecord('_grist_Tables', row_ids, values)
|
|
|
|
def create_tab_bar_action(views_to_table):
|
|
row_ids = list(xrange(1, len(views_to_table) + 1))
|
|
return actions.ReplaceTableData('_grist_TabBar', row_ids, {
|
|
'viewRef': sorted(views_to_table.keys())
|
|
})
|
|
|
|
def create_table_views_action(views_to_table, primary_views):
|
|
related_views = sorted(set(views_to_table.keys()) - set(primary_views.values()))
|
|
row_ids = list(xrange(1, len(related_views) + 1))
|
|
return actions.ReplaceTableData('_grist_TableViews', row_ids, {
|
|
'tableRef': [views_to_table[v] for v in related_views],
|
|
'viewRef': related_views,
|
|
})
|
|
|
|
return tdset.apply_doc_actions([
|
|
actions.AddTable('_grist_TabBar', [
|
|
schema.make_column('viewRef', 'Ref:_grist_Views'),
|
|
]),
|
|
actions.AddTable('_grist_TableViews', [
|
|
schema.make_column('tableRef', 'Ref:_grist_Tables'),
|
|
schema.make_column('viewRef', 'Ref:_grist_Views'),
|
|
]),
|
|
add_column('_grist_Tables', 'primaryViewId', 'Ref:_grist_Views'),
|
|
create_primary_views_action(primary_views),
|
|
create_tab_bar_action(views_to_table),
|
|
create_table_views_action(views_to_table, primary_views)
|
|
])
|
|
|
|
@migration(schema_version=3)
|
|
def migration3(tdset):
|
|
"""
|
|
There is no longer a "Derived" type for columns, and summary tables use the type suitable for
|
|
the column being summarized. For old documents, convert "Derived" type to "Any", and adjust the
|
|
usage of "lookupOrAddDerived()" function.
|
|
"""
|
|
# Note that this is a complicated migration, and mainly acceptable because it is before our very
|
|
# first release. For a released product, a change like this should be done in a backwards
|
|
# compatible way: keep but deprecate 'Derived'; introduce a lookupOrAddDerived2() to use for new
|
|
# summary tables, but keep the old interface as well for existing ones. The reason is that such
|
|
# migrations are error-prone and may mess up customers' data.
|
|
doc_actions = []
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables']))
|
|
tables_map = {t.id: t for t in tables}
|
|
columns = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables_column']))
|
|
|
|
# Convert columns from type 'Derived' to type 'Any'
|
|
affected_cols = [c for c in columns if c.type == 'Derived']
|
|
if affected_cols:
|
|
doc_actions.extend(
|
|
actions.ModifyColumn(tables_map[c.parentId].tableId, c.colId, {'type': 'Any'})
|
|
for c in affected_cols
|
|
)
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_Tables_column',
|
|
[c.id for c in affected_cols],
|
|
{'type': ['Any' for c in affected_cols]}
|
|
))
|
|
|
|
# Convert formulas of the form '.lookupOrAddDerived($x,$y)' to '.lookupOrAddDerived(x=$x,y=$y)'
|
|
formula_re = re.compile(r'(\w+).lookupOrAddDerived\((.*?)\)')
|
|
arg_re = re.compile(r'^\$(\w+)$')
|
|
def replace(match):
|
|
args = ", ".join(arg_re.sub(r'\1=$\1', arg.strip()) for arg in match.group(2).split(","))
|
|
return '%s.lookupOrAddDerived(%s)' % (match.group(1), args)
|
|
|
|
formula_updates = []
|
|
for c in columns:
|
|
new_formula = c.formula and formula_re.sub(replace, c.formula)
|
|
if new_formula != c.formula:
|
|
formula_updates.append((c, new_formula))
|
|
|
|
if formula_updates:
|
|
doc_actions.extend(
|
|
actions.ModifyColumn(tables_map[c.parentId].tableId, c.colId, {'formula': f})
|
|
for c, f in formula_updates
|
|
)
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_Tables_column',
|
|
[c.id for c, f in formula_updates],
|
|
{'formula': [f for c, f in formula_updates]}
|
|
))
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=4)
|
|
def migration4(tdset):
|
|
"""
|
|
Add TabPos column to TabBar table
|
|
"""
|
|
doc_actions = []
|
|
row_ids = tdset.all_tables['_grist_TabBar'].row_ids
|
|
doc_actions.append(add_column('_grist_TabBar', 'tabPos', 'PositionNumber'))
|
|
doc_actions.append(actions.BulkUpdateRecord('_grist_TabBar', row_ids, {'tabPos': row_ids}))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=5)
|
|
def migration5(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_Views', 'primaryViewTable', 'Ref:_grist_Tables',
|
|
formula='_grist_Tables.lookupOne(primaryViewId=$id)', isFormula=True),
|
|
])
|
|
|
|
@migration(schema_version=6)
|
|
def migration6(tdset):
|
|
# This undoes the previous migration, since primaryViewTable is now a formula private to the
|
|
# sandbox rather than part of the document schema.
|
|
return tdset.apply_doc_actions([
|
|
actions.RemoveColumn('_grist_Views', 'primaryViewTable'),
|
|
])
|
|
|
|
@migration(schema_version=7)
|
|
def migration7(tdset):
|
|
"""
|
|
Add summarySourceTable/summarySourceCol fields to metadata, and adjust existing summary tables
|
|
to correspond to the new style.
|
|
"""
|
|
# Note: this migration has some faults.
|
|
# - It doesn't delete viewSectionFields for columns it removes (if a user added some special
|
|
# columns manually.
|
|
# - It doesn't fix types of Reference columns that refer to old-style summary tables
|
|
# (if the user created some such columns manually).
|
|
|
|
doc_actions = [action for action in [
|
|
maybe_add_column(tdset, '_grist_Tables', 'summarySourceTable', 'Ref:_grist_Tables'),
|
|
maybe_add_column(tdset, '_grist_Tables_column', 'summarySourceCol', 'Ref:_grist_Tables_column')
|
|
] if action]
|
|
|
|
# Maps tableRef to Table object.
|
|
tables_map = {t.id: t for t in actions.transpose_bulk_action(tdset.all_tables['_grist_Tables'])}
|
|
|
|
# Maps tableName to tableRef
|
|
table_name_to_ref = {t.tableId: t.id for t in six.itervalues(tables_map)}
|
|
|
|
# List of Column objects
|
|
columns = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables_column']))
|
|
|
|
# Maps columnRef to Column object.
|
|
columns_map_by_ref = {c.id: c for c in columns}
|
|
|
|
# Maps (tableRef, colName) to Column object.
|
|
columns_map_by_table_colid = {(c.parentId, c.colId): c for c in columns}
|
|
|
|
# Set of all tableNames.
|
|
table_name_set = set(table_name_to_ref.keys())
|
|
|
|
remove_cols = [] # List of columns to remove
|
|
formula_updates = [] # List of (column, new_table_name, new_formula) pairs
|
|
table_renames = [] # List of (table, new_name) pairs
|
|
source_tables = [] # List of (table, summarySourceTable) pairs
|
|
source_cols = [] # List of (column, summarySourceColumn) pairs
|
|
|
|
# Summary tables used to be named as "Summary_<SourceName>_<ColRef1>_<ColRef2>". This regular
|
|
# expression parses that.
|
|
summary_re = re.compile(r'^Summary_(\w+?)((?:_\d+)*)$')
|
|
for t in six.itervalues(tables_map):
|
|
m = summary_re.match(t.tableId)
|
|
if not m or m.group(1) not in table_name_to_ref:
|
|
continue
|
|
# We have a valid summary table.
|
|
source_table_name = m.group(1)
|
|
source_table_ref = table_name_to_ref[source_table_name]
|
|
groupby_colrefs = [int(x) for x in m.group(2).strip("_").split("_")]
|
|
# Prepare a new-style name for the summary table. Be sure not to conflict with existing tables
|
|
# or with each other (i.e. don't rename multiple tables to the same name).
|
|
groupby_col_ids = [columns_map_by_ref[c].colId for c in groupby_colrefs]
|
|
new_name = summary.encode_summary_table_name(source_table_name, groupby_col_ids)
|
|
new_name = identifiers.pick_table_ident(new_name, avoid=table_name_set)
|
|
table_name_set.add(new_name)
|
|
log.warn("Upgrading summary table %s for %s(%s) to %s" % (
|
|
t.tableId, source_table_name, groupby_colrefs, new_name))
|
|
|
|
# Remove the "lookupOrAddDerived" column from the source table (which is named using the
|
|
# summary table name for its colId).
|
|
remove_cols.extend(c for c in columns
|
|
if c.parentId == source_table_ref and c.colId == t.tableId)
|
|
|
|
# Upgrade the "group" formula in the summary table.
|
|
expected_group_formula = "%s.lookupRecords(%s=$id)" % (source_table_name, t.tableId)
|
|
new_formula = "table.getSummarySourceGroup(rec)"
|
|
formula_updates.extend((c, new_name, new_formula) for c in columns
|
|
if (c.parentId == t.id and c.colId == "group" and
|
|
c.formula == expected_group_formula))
|
|
|
|
# Schedule a rename of the summary table.
|
|
table_renames.append((t, new_name))
|
|
|
|
# Set summarySourceTable fields on the metadata.
|
|
source_tables.append((t, source_table_ref))
|
|
|
|
# Set summarySourceCol fields in the metadata. We need to find the right summary column.
|
|
groupby_cols = set()
|
|
for col_ref in groupby_colrefs:
|
|
src_col = columns_map_by_ref.get(col_ref)
|
|
sum_col = columns_map_by_table_colid.get((t.id, src_col.colId)) if src_col else None
|
|
if sum_col:
|
|
groupby_cols.add(sum_col)
|
|
source_cols.append((sum_col, src_col.id))
|
|
else:
|
|
log.warn("Upgrading summary table %s: couldn't find column %s" % (t.tableId, col_ref))
|
|
|
|
# Finally, we have to remove all non-formula columns that are not groupby-columns (e.g.
|
|
# 'manualSort'), because the new approach assumes ALL non-formula columns are for groupby.
|
|
remove_cols.extend(c for c in columns
|
|
if c.parentId == t.id and c not in groupby_cols and not c.isFormula)
|
|
|
|
# Create all the doc actions from the arrays we prepared.
|
|
|
|
# Process remove_cols
|
|
doc_actions.extend(
|
|
actions.RemoveColumn(tables_map[c.parentId].tableId, c.colId) for c in remove_cols)
|
|
doc_actions.append(actions.BulkRemoveRecord(
|
|
'_grist_Tables_column', [c.id for c in remove_cols]))
|
|
|
|
# Process table_renames
|
|
doc_actions.extend(
|
|
actions.RenameTable(t.tableId, new) for (t, new) in table_renames)
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_Tables', [t.id for t, new in table_renames],
|
|
{'tableId': [new for t, new in table_renames]}
|
|
))
|
|
|
|
# Process source_tables and source_cols
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_Tables', [t.id for t, ref in source_tables],
|
|
{'summarySourceTable': [ref for t, ref in source_tables]}
|
|
))
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_Tables_column', [t.id for t, ref in source_cols],
|
|
{'summarySourceCol': [ref for t, ref in source_cols]}
|
|
))
|
|
|
|
# Process formula_updates. Do this last since recalculation of these may cause new records added
|
|
# to summary tables, so we should have all the tables correctly set up by this time.
|
|
doc_actions.extend(
|
|
actions.ModifyColumn(table_id, c.colId, {'formula': f})
|
|
for c, table_id, f in formula_updates)
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_Tables_column', [c.id for c, t, f in formula_updates],
|
|
{'formula': [f for c, t, f in formula_updates]}
|
|
))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=8)
|
|
def migration8(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_Tables_column', 'untieColIdFromLabel', 'Bool'),
|
|
])
|
|
|
|
@migration(schema_version=9)
|
|
def migration9(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_Tables_column', 'displayCol', 'Ref:_grist_Tables_column'),
|
|
add_column('_grist_Views_section_field', 'displayCol', 'Ref:_grist_Tables_column'),
|
|
])
|
|
|
|
@migration(schema_version=10)
|
|
def migration10(tdset):
|
|
"""
|
|
Add displayCol to all reference cols, with formula $<ref_col_id>.<visible_col_id>
|
|
(Note that displayCol field was added in the previous migration.)
|
|
"""
|
|
doc_actions = []
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables']))
|
|
columns = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables_column']))
|
|
|
|
# Maps tableRef to tableId.
|
|
tables_map = {t.id: t.tableId for t in tables}
|
|
|
|
# Maps tableRef to sets of colIds in the tables. Used to prevent repeated colIds.
|
|
table_col_ids = {t.id: set(tdset.all_tables[t.tableId].columns.keys()) for t in tables}
|
|
|
|
# Get the next sequential column row id.
|
|
row_id = next_id(tdset, '_grist_Tables_column')
|
|
|
|
for c in columns:
|
|
# If a column is a reference with an unset display column, add a display column.
|
|
if c.type.startswith('Ref:') and not c.displayCol:
|
|
# Get visible_col_id. If not found, row id is used and no display col is necessary.
|
|
visible_col_id = ""
|
|
try:
|
|
visible_col_id = json.loads(c.widgetOptions).get('visibleCol')
|
|
if not visible_col_id:
|
|
continue
|
|
except Exception:
|
|
continue # If invalid widgetOptions, skip this column.
|
|
|
|
# Set formula to use the current visibleCol in widgetOptions.
|
|
formula = ("$%s.%s" % (c.colId, visible_col_id))
|
|
|
|
# Get a unique colId for the display column, and add it to the set of used ids.
|
|
used_col_ids = table_col_ids[c.parentId]
|
|
display_col_id = identifiers.pick_col_ident('gristHelper_Display', avoid=used_col_ids)
|
|
used_col_ids.add(display_col_id)
|
|
|
|
# Add all actions to the list.
|
|
doc_actions.append(add_column(tables_map[c.parentId], 'gristHelper_Display', 'Any',
|
|
formula=formula, isFormula=True))
|
|
doc_actions.append(actions.AddRecord('_grist_Tables_column', row_id, {
|
|
'parentPos': 1.0,
|
|
'label': 'gristHelper_Display',
|
|
'isFormula': True,
|
|
'parentId': c.parentId,
|
|
'colId': 'gristHelper_Display',
|
|
'formula': formula,
|
|
'widgetOptions': '',
|
|
'type': 'Any'
|
|
}))
|
|
doc_actions.append(actions.UpdateRecord('_grist_Tables_column', c.id, {'displayCol': row_id}))
|
|
|
|
# Increment row id to the next unused.
|
|
row_id += 1
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=11)
|
|
def migration11(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_Views_section', 'embedId', 'Text'),
|
|
])
|
|
|
|
@migration(schema_version=12)
|
|
def migration12(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_Views_section', 'options', 'Text')
|
|
])
|
|
|
|
@migration(schema_version=13)
|
|
def migration13(tdset):
|
|
# Adds a basketId to the entire document to take advantage of basket functionality.
|
|
# From this version on, embedId is deprecated.
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_DocInfo', 'basketId', 'Text')
|
|
])
|
|
|
|
@migration(schema_version=14)
|
|
def migration14(tdset):
|
|
# Create the ACL table AND also the default ACL groups, default resource, and the default rule.
|
|
# These match the actions applied to new document by 'InitNewDoc' useraction (as of v14).
|
|
return tdset.apply_doc_actions([
|
|
actions.AddTable('_grist_ACLMemberships', [
|
|
schema.make_column('parent', 'Ref:_grist_ACLPrincipals'),
|
|
schema.make_column('child', 'Ref:_grist_ACLPrincipals'),
|
|
]),
|
|
actions.AddTable('_grist_ACLPrincipals', [
|
|
schema.make_column('userName', 'Text'),
|
|
schema.make_column('groupName', 'Text'),
|
|
schema.make_column('userEmail', 'Text'),
|
|
schema.make_column('instanceId', 'Text'),
|
|
schema.make_column('type', 'Text'),
|
|
]),
|
|
actions.AddTable('_grist_ACLResources', [
|
|
schema.make_column('colIds', 'Text'),
|
|
schema.make_column('tableId', 'Text'),
|
|
]),
|
|
actions.AddTable('_grist_ACLRules', [
|
|
schema.make_column('aclFormula', 'Text'),
|
|
schema.make_column('principals', 'Text'),
|
|
schema.make_column('resource', 'Ref:_grist_ACLResources'),
|
|
schema.make_column('aclColumn', 'Ref:_grist_Tables_column'),
|
|
schema.make_column('permissions', 'Int'),
|
|
]),
|
|
|
|
# Set up initial ACL data.
|
|
actions.BulkAddRecord('_grist_ACLPrincipals', [1,2,3,4], {
|
|
'type': ['group', 'group', 'group', 'group'],
|
|
'groupName': ['Owners', 'Admins', 'Editors', 'Viewers'],
|
|
}),
|
|
actions.AddRecord('_grist_ACLResources', 1, {
|
|
'tableId': '', 'colIds': ''
|
|
}),
|
|
actions.AddRecord('_grist_ACLRules', 1, {
|
|
'resource': 1, 'permissions': 0x3F, 'principals': '[1]'
|
|
}),
|
|
])
|
|
|
|
@migration(schema_version=15)
|
|
def migration15(tdset):
|
|
# Adds a filter JSON property to each field.
|
|
# From this version on, filterSpec in _grist_Views_section is deprecated.
|
|
doc_actions = [
|
|
add_column('_grist_Views_section_field', 'filter', 'Text')
|
|
]
|
|
|
|
# Get all section and field data to move section filter data to the fields
|
|
sections = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Views_section']))
|
|
fields = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Views_section_field']))
|
|
|
|
specs = {s.id: safe_parse(s.filterSpec) for s in sections}
|
|
|
|
# Move filter data from sections to fields
|
|
for f in fields:
|
|
# If the field belongs to the section and the field's colRef is in its filterSpec,
|
|
# pull the filter setting from the section.
|
|
filter_spec = specs.get(f.parentId)
|
|
if filter_spec and str(f.colRef) in filter_spec:
|
|
doc_actions.append(actions.UpdateRecord('_grist_Views_section_field', f.id, {
|
|
'filter': json.dumps(filter_spec[str(f.colRef)])
|
|
}))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=16)
|
|
def migration16(tdset):
|
|
# Add visibleCol to columns and view fields, and set it from columns' and fields' widgetOptions.
|
|
doc_actions = [
|
|
add_column('_grist_Tables_column', 'visibleCol', 'Ref:_grist_Tables_column'),
|
|
add_column('_grist_Views_section_field', 'visibleCol', 'Ref:_grist_Tables_column'),
|
|
]
|
|
|
|
# Maps tableId to table, for looking up target table as listed in "Ref:*" types.
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables']))
|
|
tables_by_id = {t.tableId: t for t in tables}
|
|
|
|
# Allow looking up columns by ref or by (tableRef, colId)
|
|
columns = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables_column']))
|
|
columns_by_ref = {c.id: c for c in columns}
|
|
columns_by_id = {(c.parentId, c.colId): c.id for c in columns}
|
|
|
|
# Helper which returns the {'visibleCol', 'widgetOptions'} update visibleCol should be set.
|
|
def convert_visible_col(col, widget_options):
|
|
if not col.type.startswith('Ref:'):
|
|
return None
|
|
|
|
# To set visibleCol, we need to know the target table. Skip if we can't find it.
|
|
target_table = tables_by_id.get(col.type[len('Ref:'):])
|
|
if not target_table:
|
|
return None
|
|
|
|
try:
|
|
parsed_options = json.loads(widget_options)
|
|
except Exception:
|
|
return None # If invalid widgetOptions, skip this column.
|
|
|
|
visible_col_id = parsed_options.pop('visibleCol', None)
|
|
if not visible_col_id:
|
|
return None
|
|
|
|
# Find visible_col_id as the column name in the appropriate table.
|
|
target_col_ref = (0 if visible_col_id == 'id' else
|
|
columns_by_id.get((target_table.id, visible_col_id), None))
|
|
if target_col_ref is None:
|
|
return None
|
|
|
|
# Use compact separators without whitespace, to match how JS encodes JSON.
|
|
return {'visibleCol': target_col_ref,
|
|
'widgetOptions': json.dumps(parsed_options, separators=(',', ':')) }
|
|
|
|
for c in columns:
|
|
new_values = convert_visible_col(c, c.widgetOptions)
|
|
if new_values:
|
|
doc_actions.append(actions.UpdateRecord('_grist_Tables_column', c.id, new_values))
|
|
|
|
fields = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Views_section_field']))
|
|
for f in fields:
|
|
c = columns_by_ref.get(f.colRef)
|
|
if c:
|
|
new_values = convert_visible_col(c, f.widgetOptions)
|
|
if new_values:
|
|
doc_actions.append(actions.UpdateRecord('_grist_Views_section_field', f.id, new_values))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
# This is actually the only migration that requires all tables because it modifies user data
|
|
# (specifically, any columns of the deprecated "Image" type).
|
|
@migration(schema_version=17, need_all_tables=True)
|
|
def migration17(tdset):
|
|
"""
|
|
There is no longer an "Image" type for columns, as "Attachments" now serves as a
|
|
display type for arbitrary files including images. Convert "Image" columns to "Attachments"
|
|
columns.
|
|
"""
|
|
doc_actions = []
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables']))
|
|
tables_map = {t.id: t for t in tables}
|
|
columns = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables_column']))
|
|
|
|
# Convert columns from type 'Image' to type 'Attachments'
|
|
affected_cols = [c for c in columns if c.type == 'Image']
|
|
conv = lambda val: [val] if isinstance(val, int) and val > 0 else []
|
|
if affected_cols:
|
|
# Update the types in the data tables
|
|
doc_actions.extend(
|
|
actions.ModifyColumn(tables_map[c.parentId].tableId, c.colId, {'type': 'Attachments'})
|
|
for c in affected_cols
|
|
)
|
|
# Update the types in the metadata tables
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_Tables_column',
|
|
[c.id for c in affected_cols],
|
|
{'type': ['Attachments' for c in affected_cols]}
|
|
))
|
|
# Update the values to lists
|
|
for c in affected_cols:
|
|
if c.isFormula:
|
|
# Formula columns don't have data stored in DB, should not have data changes.
|
|
continue
|
|
table_id = tables_map[c.parentId].tableId
|
|
table = tdset.all_tables[table_id]
|
|
doc_actions.append(
|
|
actions.BulkUpdateRecord(table_id, table.row_ids,
|
|
{c.colId: [conv(val) for val in table.columns[c.colId]]})
|
|
)
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=18)
|
|
def migration18(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_DocInfo', 'timezone', 'Text'),
|
|
# all documents prior to this migration have been created in New York
|
|
actions.UpdateRecord('_grist_DocInfo', 1, {'timezone': 'America/New_York'})
|
|
])
|
|
|
|
@migration(schema_version=19)
|
|
def migration19(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_Tables', 'onDemand', 'Bool'),
|
|
])
|
|
|
|
@migration(schema_version=20)
|
|
def migration20(tdset):
|
|
"""
|
|
Add _grist_Pages table and populate based on existing TableViews entries, ie: tables are sorted
|
|
alphabetically by their `tableId` and views are gathered within their corresponding table and
|
|
sorted by their id.
|
|
"""
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables']))
|
|
table_map = {t.id: t for t in tables}
|
|
table_views = list(actions.transpose_bulk_action(tdset.all_tables['_grist_TableViews']))
|
|
# Old docs may include "Other views", not associated with any table. Don't include those in
|
|
# table_views_map: they'll get included but not sorted or grouped by tableId.
|
|
table_views_map = {tv.viewRef: table_map[tv.tableRef].tableId
|
|
for tv in table_views if tv.tableRef in table_map}
|
|
views = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Views']))
|
|
def view_key(view):
|
|
"""
|
|
Returns ("Table1", 2) where "Table1" is the view's tableId and 2 the view id. For
|
|
primary view (ie: not referenced in _grist_TableViews) returns ("Table1", -1). Useful
|
|
to get the list of views sorted in the same way as in the Table side pane. We use -1
|
|
for primary view to make sure they come first among all the views of the same table.
|
|
"""
|
|
if view.id in table_views_map:
|
|
return (table_views_map[view.id], view.id)
|
|
# the name of primary view's is the same as the tableId
|
|
return (view.name, -1)
|
|
views.sort(key=view_key)
|
|
row_ids = list(xrange(1, len(views) + 1))
|
|
return tdset.apply_doc_actions([
|
|
actions.AddTable('_grist_Pages', [
|
|
schema.make_column('viewRef', 'Ref:_grist_Views'),
|
|
schema.make_column('pagePos', 'PositionNumber'),
|
|
schema.make_column('indentation', 'Int'),
|
|
]),
|
|
actions.ReplaceTableData('_grist_Pages', row_ids, {
|
|
'viewRef': [v.id for v in views],
|
|
'pagePos': row_ids,
|
|
'indentation': [1 if v.id in table_views_map else 0 for v in views]
|
|
})
|
|
])
|
|
|
|
@migration(schema_version=21)
|
|
def migration21(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_ACLRules', 'aclFormulaParsed', 'Text'),
|
|
add_column('_grist_ACLRules', 'permissionsText', 'Text'),
|
|
add_column('_grist_ACLRules', 'rulePos', 'PositionNumber'),
|
|
add_column('_grist_ACLRules', 'userAttributes', 'Text'),
|
|
])
|
|
|
|
|
|
@migration(schema_version=22)
|
|
def migration22(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_Tables_column', 'recalcWhen', 'Int'),
|
|
add_column('_grist_Tables_column', 'recalcDeps', 'RefList:_grist_Tables_column'),
|
|
])
|
|
|
|
@migration(schema_version=23)
|
|
def migration23(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_DocInfo', 'documentSettings', 'Text'),
|
|
actions.UpdateRecord('_grist_DocInfo', 1, {'documentSettings': '{"locale":"en-US"}'})
|
|
])
|
|
|
|
|
|
@migration(schema_version=24)
|
|
def migration24(tdset):
|
|
return tdset.apply_doc_actions([
|
|
actions.AddTable('_grist_Triggers', [
|
|
schema.make_column("tableRef", "Ref:_grist_Tables"),
|
|
schema.make_column("eventTypes", "ChoiceList"),
|
|
schema.make_column("isReadyColRef", "Ref:_grist_Tables_column"),
|
|
schema.make_column("actions", "Text"), # JSON
|
|
]),
|
|
])
|
|
|
|
@migration(schema_version=25)
|
|
def migration25(tdset):
|
|
"""
|
|
Add _grist_Filters table and populate based on existing filters stored
|
|
in _grist_Views_section_field.
|
|
|
|
From this version on, filter in _grist_Views_section_field is deprecated.
|
|
"""
|
|
doc_actions = [
|
|
actions.AddTable('_grist_Filters', [
|
|
schema.make_column("viewSectionRef", "Ref:_grist_Views_section"),
|
|
schema.make_column("colRef", "Ref:_grist_Tables_column"),
|
|
schema.make_column("filter", "Text"),
|
|
])
|
|
]
|
|
|
|
# Move existing field filters to _grist_Filters.
|
|
fields = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Views_section_field']))
|
|
col_info = { 'filter': [], 'colRef': [], 'viewSectionRef': [] }
|
|
for f in fields:
|
|
if not f.filter:
|
|
continue
|
|
|
|
col_info['filter'].append(f.filter)
|
|
col_info['colRef'].append(f.colRef)
|
|
col_info['viewSectionRef'].append(f.parentId)
|
|
|
|
num_filters = len(col_info['filter'])
|
|
if num_filters > 0:
|
|
doc_actions.append(actions.BulkAddRecord('_grist_Filters', [None] * num_filters, col_info))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
|
|
@migration(schema_version=26)
|
|
def migration26(tdset):
|
|
"""
|
|
Add rawViewSectionRef column to _grist_Tables
|
|
and new raw view sections for each 'normal' table.
|
|
"""
|
|
doc_actions = [add_column('_grist_Tables', 'rawViewSectionRef', 'Ref:_grist_Views_section')]
|
|
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables["_grist_Tables"]))
|
|
columns = list(actions.transpose_bulk_action(tdset.all_tables["_grist_Tables_column"]))
|
|
views = {view.id: view
|
|
for view in actions.transpose_bulk_action(tdset.all_tables["_grist_Views"])}
|
|
|
|
new_view_section_id = next_id(tdset, "_grist_Views_section")
|
|
|
|
for table in sorted(tables, key=lambda t: t.tableId):
|
|
old_view = views.get(table.primaryViewId)
|
|
if not (table.primaryViewId and old_view):
|
|
continue
|
|
|
|
table_columns = [
|
|
col for col in columns
|
|
if table.id == col.parentId and is_visible_column(col.colId)
|
|
]
|
|
table_columns.sort(key=lambda c: c.parentPos)
|
|
fields = {
|
|
"parentId": [new_view_section_id] * len(table_columns),
|
|
"colRef": [col.id for col in table_columns],
|
|
"parentPos": [col.parentPos for col in table_columns],
|
|
}
|
|
field_ids = [None] * len(table_columns)
|
|
|
|
doc_actions += [
|
|
actions.AddRecord(
|
|
"_grist_Views_section", new_view_section_id, {
|
|
"tableRef": table.id,
|
|
"parentId": 0,
|
|
"parentKey": "record",
|
|
"title": old_view.name,
|
|
"defaultWidth": 100,
|
|
"borderWidth": 1,
|
|
}),
|
|
actions.UpdateRecord(
|
|
"_grist_Tables", table.id, {
|
|
"rawViewSectionRef": new_view_section_id,
|
|
}),
|
|
actions.BulkAddRecord(
|
|
"_grist_Views_section_field", field_ids, fields
|
|
),
|
|
]
|
|
|
|
new_view_section_id += 1
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
|
|
@migration(schema_version=27)
|
|
def migration27(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_Tables_column', 'rules', 'RefList:_grist_Tables_column'),
|
|
add_column('_grist_Views_section_field', 'rules', 'RefList:_grist_Tables_column'),
|
|
])
|
|
|
|
|
|
@migration(schema_version=28)
|
|
def migration28(tdset):
|
|
doc_actions = [add_column('_grist_Attachments', 'timeDeleted', 'DateTime')]
|
|
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables["_grist_Tables"]))
|
|
columns = list(actions.transpose_bulk_action(tdset.all_tables["_grist_Tables_column"]))
|
|
|
|
for table in tables:
|
|
for col in columns:
|
|
if table.id == col.parentId and col.type == "Attachments":
|
|
# This looks like it doesn't change anything,
|
|
# but it makes DocStorage realise that the sqlType has changed
|
|
# so it converts marshalled blobs to JSON
|
|
doc_actions.append(actions.ModifyColumn(table.tableId, col.colId, {"type": "Attachments"}))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
|
|
@migration(schema_version=29)
|
|
def migration29(tdset):
|
|
# This migration is fixing an error on summary tables with conditional rules.
|
|
# On summary tables all formula columns with the same name were updated together,
|
|
# which caused a situation where some summary columns have rules from diffrent tables.
|
|
# This migration is removing those rules in such columns.
|
|
|
|
tables = {table.id: table
|
|
for table in actions.transpose_bulk_action(tdset.all_tables["_grist_Tables"])}
|
|
columns = {col.id: col
|
|
for col in actions.transpose_bulk_action(tdset.all_tables["_grist_Tables_column"])}
|
|
doc_actions = []
|
|
|
|
def is_valid_rule(parentId, rule_id):
|
|
# Valid rule should be an existing column,
|
|
rule_col = columns.get(rule_id)
|
|
# in the same table.
|
|
return rule_col and rule_col.parentId == parentId
|
|
|
|
for col in columns.values():
|
|
if col.rules:
|
|
# Parse rules (they are a json encoded array like '[15]')
|
|
rules = safe_parse(col.rules)
|
|
# Remove all conditional styles if anything about rules is invalid.
|
|
if not (
|
|
isinstance(rules, list) and
|
|
all(is_valid_rule(col.parentId, ruleId) for ruleId in rules)
|
|
):
|
|
doc_actions.append(actions.UpdateRecord('_grist_Tables_column', col.id, {
|
|
"rules": None,
|
|
"widgetOptions": summary._copy_widget_options(col.widgetOptions)
|
|
}))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=30)
|
|
def migration30(tdset):
|
|
"""
|
|
Add raw view sections for each summary table. This is similar to migration 26, but for
|
|
summary tables instead of user tables.
|
|
"""
|
|
doc_actions = []
|
|
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables["_grist_Tables"]))
|
|
columns = list(actions.transpose_bulk_action(tdset.all_tables["_grist_Tables_column"]))
|
|
|
|
new_view_section_id = next_id(tdset, "_grist_Views_section")
|
|
|
|
for table in sorted(tables, key=lambda t: t.tableId):
|
|
if not table.summarySourceTable:
|
|
continue
|
|
|
|
table_columns = [
|
|
col for col in columns
|
|
if table.id == col.parentId and is_visible_column(col.colId)
|
|
]
|
|
table_columns.sort(key=lambda c: c.parentPos)
|
|
fields = {
|
|
"parentId": [new_view_section_id] * len(table_columns),
|
|
"colRef": [col.id for col in table_columns],
|
|
"parentPos": [col.parentPos for col in table_columns],
|
|
}
|
|
field_ids = [None] * len(table_columns)
|
|
|
|
doc_actions += [
|
|
actions.AddRecord(
|
|
"_grist_Views_section", new_view_section_id, {
|
|
"tableRef": table.id,
|
|
"parentId": 0,
|
|
"parentKey": "record",
|
|
"title": "",
|
|
"defaultWidth": 100,
|
|
"borderWidth": 1,
|
|
}
|
|
),
|
|
actions.UpdateRecord(
|
|
"_grist_Tables", table.id, {
|
|
"rawViewSectionRef": new_view_section_id,
|
|
})
|
|
,
|
|
actions.BulkAddRecord(
|
|
"_grist_Views_section_field", field_ids, fields
|
|
),
|
|
]
|
|
|
|
new_view_section_id += 1
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
|
|
@migration(schema_version=31)
|
|
def migration31(tdset):
|
|
columns = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables_column']))
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables']))
|
|
acl_resources = list(actions.transpose_bulk_action(tdset.all_tables['_grist_ACLResources']))
|
|
|
|
tables_by_ref = {t.id: t for t in tables}
|
|
columns_by_table_ref = defaultdict(list)
|
|
for col in columns:
|
|
columns_by_table_ref[col.parentId].append(col)
|
|
|
|
table_name_set = {t.tableId for t in tables}
|
|
|
|
table_renames = [] # List of (table, new_name) pairs
|
|
|
|
for t in six.itervalues(tables_by_ref):
|
|
if not t.summarySourceTable:
|
|
continue
|
|
source_table = tables_by_ref[t.summarySourceTable]
|
|
# Prepare a new-style name for the summary table. Be sure not to conflict with existing tables
|
|
# or with each other (i.e. don't rename multiple tables to the same name).
|
|
groupby_col_ids = [c.colId for c in columns_by_table_ref[t.id] if c.summarySourceCol]
|
|
new_name = summary.encode_summary_table_name(source_table.tableId, groupby_col_ids)
|
|
if new_name == t.tableId:
|
|
continue
|
|
new_name = identifiers.pick_table_ident(new_name, avoid=table_name_set)
|
|
table_name_set.add(new_name)
|
|
log.warn("Upgrading summary table %s for %s(%s) to %s" % (
|
|
t.tableId, source_table.tableId, groupby_col_ids, new_name))
|
|
|
|
# Schedule a rename of the summary table.
|
|
table_renames.append((t, new_name))
|
|
|
|
doc_actions = [
|
|
actions.RenameTable(t.tableId, new)
|
|
for (t, new) in table_renames
|
|
]
|
|
if table_renames:
|
|
doc_actions.append(
|
|
actions.BulkUpdateRecord(
|
|
'_grist_Tables', [t.id for t, new in table_renames],
|
|
{'tableId': [new for t, new in table_renames]}
|
|
)
|
|
)
|
|
|
|
# Update formulas in all columns containing old-style names like 'GristSummary_'
|
|
for col in columns:
|
|
if 'GristSummary_' not in col.formula:
|
|
continue
|
|
formula = col.formula
|
|
for table, new_name in table_renames:
|
|
# Use regex to only match whole words
|
|
formula = re.sub(r'\b%s\b' % table.tableId, new_name, formula)
|
|
doc_actions.append(actions.UpdateRecord('_grist_Tables_column', col.id, {'formula': formula}))
|
|
|
|
table_renames_dict = {t.tableId: new for t, new in table_renames}
|
|
for resource in acl_resources:
|
|
new_name = table_renames_dict.get(resource.tableId)
|
|
if new_name:
|
|
doc_actions.append(
|
|
actions.UpdateRecord('_grist_ACLResources', resource.id, {'tableId': new_name})
|
|
)
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=32)
|
|
def migration32(tdset):
|
|
return tdset.apply_doc_actions([
|
|
add_column('_grist_Views_section', 'rules', 'RefList:_grist_Tables_column'),
|
|
])
|
|
|
|
@migration(schema_version=33)
|
|
def migration33(tdset):
|
|
"""
|
|
Add _grist_Cells table
|
|
"""
|
|
doc_actions = [
|
|
actions.AddTable('_grist_Cells', [
|
|
schema.make_column("tableRef", "Ref:_grist_Tables"),
|
|
schema.make_column("colRef", "Ref:_grist_Tables_column"),
|
|
schema.make_column("rowId", "Int"),
|
|
schema.make_column("root", "Bool"),
|
|
schema.make_column("parentId", "Ref:_grist_Cells"),
|
|
schema.make_column("type", "Int"),
|
|
schema.make_column("content", "Text"),
|
|
schema.make_column("userRef", "Text"),
|
|
]),
|
|
]
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=34)
|
|
def migration34(tdset):
|
|
"""
|
|
Add pinned column to _grist_Filters and populate based on existing sections.
|
|
|
|
When populating, pinned will be set to true for filters that either belong to
|
|
a section where the filter bar is toggled or a raw view section.
|
|
|
|
From this version on, _grist_Views_section.options.filterBar is deprecated.
|
|
"""
|
|
doc_actions = [add_column('_grist_Filters', 'pinned', 'Bool')]
|
|
|
|
tables = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Tables']))
|
|
sections = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Views_section']))
|
|
filters = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Filters']))
|
|
raw_section_ids = set(t.rawViewSectionRef for t in tables)
|
|
filter_bar_by_section_id = {
|
|
# Pre-migration, raw sections always showed the filter bar in the UI. Since we want
|
|
# existing raw section filters to continue appearing in the filter bar, we'll pretend
|
|
# here that raw sections have a filterBar value of True. Note that after this migration
|
|
# it will be possible for raw sections to have unpinned filters.
|
|
s.id: bool(s.id in raw_section_ids or safe_parse(s.options).get('filterBar', False))
|
|
for s in sections
|
|
}
|
|
|
|
# List of (filter_rec, pinned) pairs.
|
|
filter_updates = []
|
|
for filter_rec in filters:
|
|
filter_updates.append((
|
|
filter_rec,
|
|
filter_bar_by_section_id.get(filter_rec.viewSectionRef, False)
|
|
))
|
|
|
|
if filter_updates:
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_Filters',
|
|
[filter_rec.id for filter_rec, _ in filter_updates],
|
|
{'pinned': [pinned for _, pinned in filter_updates]},
|
|
))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=35)
|
|
def migration35(tdset):
|
|
"""
|
|
Add memo column to _grist_ACLRules and populate with comments stored in
|
|
_grist_ACLRules.aclFormula.
|
|
|
|
From this version on, comments in _grist_ACLRules.aclFormula will no longer
|
|
be used as memos.
|
|
"""
|
|
doc_actions = [add_column('_grist_ACLRules', 'memo', 'Text')]
|
|
|
|
acl_rules = list(actions.transpose_bulk_action(tdset.all_tables['_grist_ACLRules']))
|
|
|
|
# List of (acl_rule_rec, memo) pairs.
|
|
acl_rule_updates = []
|
|
for acl_rule_rec in acl_rules:
|
|
acl_formula = safe_parse(acl_rule_rec.aclFormulaParsed)
|
|
if not acl_formula or acl_formula[0] != 'Comment':
|
|
continue
|
|
|
|
acl_rule_updates.append((
|
|
acl_rule_rec,
|
|
acl_formula[2]
|
|
))
|
|
|
|
if acl_rule_updates:
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_ACLRules',
|
|
[acl_rule_rec.id for acl_rule_rec, _ in acl_rule_updates],
|
|
{'memo': [memo for _, memo in acl_rule_updates]},
|
|
))
|
|
|
|
return tdset.apply_doc_actions(doc_actions)
|
|
|
|
@migration(schema_version=36)
|
|
def migration36(tdset):
|
|
"""
|
|
Add description to column
|
|
"""
|
|
return tdset.apply_doc_actions([add_column('_grist_Tables_column', 'description', 'Text')])
|
|
|
|
@migration(schema_version=37)
|
|
def migration37(tdset):
|
|
"""
|
|
Add fileExt column to _grist_Attachments.
|
|
"""
|
|
return tdset.apply_doc_actions([add_column('_grist_Attachments', 'fileExt', 'Text')])
|
|
|
|
@migration(schema_version=38)
|
|
def migration38(tdset):
|
|
"""
|
|
Through a mishap, this migration ended up conflicted across two version of Grist.
|
|
In one version, it added webhook related columns. In another it added a description
|
|
to widgets. Sorry if this impacted you. Migration 39 does the best we can to
|
|
smooth over the divergence, and this migration now does nothing (though in the
|
|
past it did one of two possible things).
|
|
"""
|
|
return tdset.apply_doc_actions([])
|
|
|
|
@migration(schema_version=39)
|
|
def migration39(tdset):
|
|
"""
|
|
Adds memo, label, and enabled flag to triggers (for webhooks).
|
|
Adds a description to widgets.
|
|
"""
|
|
doc_actions = []
|
|
if 'memo' not in tdset.all_tables['_grist_Triggers'].columns:
|
|
doc_actions += [add_column('_grist_Triggers', 'memo', 'Text'),
|
|
add_column('_grist_Triggers', 'label', 'Text'),
|
|
add_column('_grist_Triggers', 'enabled', 'Bool')]
|
|
triggers = list(actions.transpose_bulk_action(tdset.all_tables['_grist_Triggers']))
|
|
doc_actions.append(actions.BulkUpdateRecord(
|
|
'_grist_Triggers',
|
|
[t.id for t in triggers],
|
|
{'enabled': [True for t in triggers]}
|
|
))
|
|
if 'description' not in tdset.all_tables['_grist_Views_section'].columns:
|
|
doc_actions.append(add_column('_grist_Views_section', 'description', 'Text'))
|
|
return tdset.apply_doc_actions(doc_actions)
|