gristlabs_grist-core/sandbox/grist/test_renames.py
Dmitry S e2226c3ab7 (core) Store formula values in DB, and include them into .stored/.undo fields of actions.
Summary:
- Introduce a new SQLiteDB migration, which adds DB columns for formula columns
- Newly added columns have the special ['P'] (pending) value in them
  (in order to show the usual "Loading..." on the first load that triggers the migration)
- Calculated values are added to .stored/.undo fields of user actions.
- Various changes made in the sandbox to include .stored/.undo in the right order.
- OnDemand tables ignore stored formula columns, replacing them with special SQL as before
- In particular, converting to OnDemand table leaves stale values in those
  columns, we should maybe clean those out.

Some tweaks on the side:
- Allow overriding chai assertion truncateThreshold with CHAI_TRUNCATE_THRESHOLD
- Rebuild python automatically in watch mode

Test Plan: Fixed various tests, updated some fixtures. Many python tests that check actions needed adjustments because actions moved from .stored to .undo. Some checks added to catch situations previously only caught in browser tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2645
2020-11-04 16:45:47 -05:00

398 lines
20 KiB
Python

# -*- coding: utf-8 -*-
import logger
import testutil
import test_engine
log = logger.Logger(__name__, logger.INFO)
class TestRenames(test_engine.EngineTestCase):
# Simpler cases of column renames in formulas. Here's the list of cases we support and test.
# $COLUMN where NAME is a column (formula or non-formula)
# $ref.COLUMN when $ref is a non-formula Reference column
# $ref.column.COLUMN
# $ref.COLUMN when $ref is a function with a Ref type.
# $ref.COLUMN when $ref is a function with Any type but clearly returning a Ref.
# Table.lookupFunc(COLUMN1=value, COLUMN2=value) and for .lookupRecords
# Table.lookupFunc(...).COLUMN and for .lookupRecords
# Table.lookupFunc(...).foo.COLUMN and for .lookupRecords
# [x.COLUMN for x in Table.lookupRecords(...)] for different kinds of comprehensions
# TABLE.lookupFunc(...) where TABLE is a user-defined table.
sample = testutil.parse_test_sample({
"SCHEMA": [
[1, "Address", [
[21, "city", "Text", False, "", "", ""],
]],
[2, "People", [
[22, "name", "Text", False, "", "", ""],
[23, "addr", "Ref:Address", False, "", "", ""],
[24, "city", "Any", True, "$addr.city", "", ""],
]]
],
"DATA": {
"Address": [
["id", "city" ],
[11, "New York" ],
[12, "Colombia" ],
[13, "New Haven" ],
[14, "West Haven" ],
],
"People": [
["id", "name" , "addr" ],
[1, "Bob" , 12 ],
[2, "Alice" , 13 ],
[3, "Doug" , 12 ],
[4, "Sam" , 11 ],
],
}
})
def test_rename_rec_attribute(self):
# Simple case: we are renaming `$COLUMN`.
self.load_sample(self.sample)
out_actions = self.apply_user_action(["RenameColumn", "People", "addr", "address"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "People", "addr", "address"],
["ModifyColumn", "People", "city", {"formula": "$address.city"}],
["BulkUpdateRecord", "_grist_Tables_column", [23, 24], {
"colId": ["address", "city"],
"formula": ["", "$address.city"]
}],
],
# Things should get recomputed, but produce same results, hence no calc actions.
"calc": []
})
# Make sure renames of formula columns are also recognized.
self.add_column("People", "CityUpper", formula="$city.upper()")
out_actions = self.apply_user_action(["RenameColumn", "People", "city", "ciudad"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "People", "city", "ciudad"],
["ModifyColumn", "People", "CityUpper", {"formula": "$ciudad.upper()"}],
["BulkUpdateRecord", "_grist_Tables_column", [24, 25], {
"colId": ["ciudad", "CityUpper"],
"formula": ["$address.city", "$ciudad.upper()"]
}]
]})
def test_rename_reference_attribute(self):
# Slightly harder: renaming `$ref.COLUMN`
self.load_sample(self.sample)
out_actions = self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "Address", "city", "ciudad"],
["ModifyColumn", "People", "city", {"formula": "$addr.ciudad"}],
["BulkUpdateRecord", "_grist_Tables_column", [21, 24], {
"colId": ["ciudad", "city"],
"formula": ["", "$addr.ciudad"]
}],
]})
def test_rename_ref_ref_attr(self):
# Slightly harder still: renaming $ref.column.COLUMN.
self.load_sample(self.sample)
self.add_column("Address", "person", type="Ref:People")
self.add_column("Address", "person_city", formula="$person.addr.city")
self.add_column("Address", "person_city2", formula="a = $person.addr\nreturn a.city")
out_actions = self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "Address", "city", "ciudad"],
["ModifyColumn", "People", "city", {"formula": "$addr.ciudad"}],
["ModifyColumn", "Address", "person_city", {"formula": "$person.addr.ciudad"}],
["ModifyColumn", "Address", "person_city2", {"formula":
"a = $person.addr\nreturn a.ciudad"}],
["BulkUpdateRecord", "_grist_Tables_column", [21, 24, 26, 27], {
"colId": ["ciudad", "city", "person_city", "person_city2"],
"formula": ["", "$addr.ciudad", "$person.addr.ciudad", "a = $person.addr\nreturn a.ciudad"]
}],
]})
def test_rename_typed_ref_func_attr(self):
# Renaming `$ref.COLUMN` when $ref is a function with a Ref type.
self.load_sample(self.sample)
self.add_column("People", "addr_func", type="Ref:Address", isFormula=True, formula="$addr")
self.add_column("People", "city2", formula="$addr_func.city")
out_actions = self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "Address", "city", "ciudad"],
["ModifyColumn", "People", "city", {"formula": "$addr.ciudad"}],
["ModifyColumn", "People", "city2", {"formula": "$addr_func.ciudad"}],
["BulkUpdateRecord", "_grist_Tables_column", [21, 24, 26], {
"colId": ["ciudad", "city", "city2"],
"formula": ["", "$addr.ciudad", "$addr_func.ciudad"]
}],
]})
def test_rename_any_ref_func_attr(self):
# Renaming `$ref.COLUMN` when $ref is a function with Any type but clearly returning a Ref.
self.load_sample(self.sample)
self.add_column("People", "addr_func", isFormula=True, formula="$addr")
self.add_column("People", "city3", formula="$addr_func.city")
out_actions = self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "Address", "city", "ciudad"],
["ModifyColumn", "People", "city", {"formula": "$addr.ciudad"}],
["ModifyColumn", "People", "city3", {"formula": "$addr_func.ciudad"}],
["BulkUpdateRecord", "_grist_Tables_column", [21, 24, 26], {
"colId": ["ciudad", "city", "city3"],
"formula": ["", "$addr.ciudad", "$addr_func.ciudad"]
}],
]})
def test_rename_reflist_attr(self):
# Renaming `$ref.COLUMN` where $ref is a data or function with RefList type (most importantly
# applies to the $group column of summary tables).
self.load_sample(self.sample)
self.add_column("People", "addr_list", type="RefList:Address", isFormula=False)
self.add_column("People", "addr_func", type="RefList:Address", isFormula=True, formula="[1,2]")
self.add_column("People", "citysum", formula="sum($addr_func.city) + sum($addr_list.city)")
out_actions = self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "Address", "city", "ciudad"],
["ModifyColumn", "People", "city", {"formula": "$addr.ciudad"}],
["ModifyColumn", "People", "citysum", {"formula":
"sum($addr_func.ciudad) + sum($addr_list.ciudad)"}],
["BulkUpdateRecord", "_grist_Tables_column", [21, 24, 27], {
"colId": ["ciudad", "city", "citysum"],
"formula": ["", "$addr.ciudad", "sum($addr_func.ciudad) + sum($addr_list.ciudad)"]
}],
]})
def test_rename_lookup_param(self):
# Renaming `Table.lookupOne(COLUMN1=value, COLUMN2=value)` and for `.lookupRecords`
self.load_sample(self.sample)
self.add_column("Address", "people", formula="People.lookupOne(addr=$id, city=$city)")
self.add_column("Address", "people2", formula="People.lookupRecords(addr=$id)")
out_actions = self.apply_user_action(["RenameColumn", "People", "addr", "ADDRESS"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "People", "addr", "ADDRESS"],
["ModifyColumn", "People", "city", {"formula": "$ADDRESS.city"}],
["ModifyColumn", "Address", "people",
{"formula": "People.lookupOne(ADDRESS=$id, city=$city)"}],
["ModifyColumn", "Address", "people2",
{"formula": "People.lookupRecords(ADDRESS=$id)"}],
["BulkUpdateRecord", "_grist_Tables_column", [23, 24, 25, 26], {
"colId": ["ADDRESS", "city", "people", "people2"],
"formula": ["", "$ADDRESS.city",
"People.lookupOne(ADDRESS=$id, city=$city)",
"People.lookupRecords(ADDRESS=$id)"]
}],
]})
# Another rename that should affect the second parameter.
out_actions = self.apply_user_action(["RenameColumn", "People", "city", "ciudad"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "People", "city", "ciudad"],
["ModifyColumn", "Address", "people",
{"formula": "People.lookupOne(ADDRESS=$id, ciudad=$city)"}],
["BulkUpdateRecord", "_grist_Tables_column", [24, 25], {
"colId": ["ciudad", "people"],
"formula": ["$ADDRESS.city", "People.lookupOne(ADDRESS=$id, ciudad=$city)"]
}],
]})
# This is kind of unnecessary, but checks how the values of params are affected separately.
out_actions = self.apply_user_action(["RenameColumn", "Address", "city", "city2"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "Address", "city", "city2"],
["ModifyColumn", "People", "ciudad", {"formula": "$ADDRESS.city2"}],
["ModifyColumn", "Address", "people",
{"formula": "People.lookupOne(ADDRESS=$id, ciudad=$city2)"}],
["BulkUpdateRecord", "_grist_Tables_column", [21, 24, 25], {
"colId": ["city2", "ciudad", "people"],
"formula": ["", "$ADDRESS.city2", "People.lookupOne(ADDRESS=$id, ciudad=$city2)"]
}],
]})
def test_rename_lookup_result_attr(self):
# Renaming `Table.lookupOne(...).COLUMN` and for `.lookupRecords`
self.load_sample(self.sample)
self.add_column("Address", "people", formula="People.lookupOne(addr=$id, city=$city).name")
self.add_column("Address", "people2", formula="People.lookupRecords(addr=$id).name")
out_actions = self.apply_user_action(["RenameColumn", "People", "name", "nombre"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "People", "name", "nombre"],
["ModifyColumn", "Address", "people", {"formula":
"People.lookupOne(addr=$id, city=$city).nombre"}],
["ModifyColumn", "Address", "people2", {"formula":
"People.lookupRecords(addr=$id).nombre"}],
["BulkUpdateRecord", "_grist_Tables_column", [22, 25, 26], {
"colId": ["nombre", "people", "people2"],
"formula": ["",
"People.lookupOne(addr=$id, city=$city).nombre",
"People.lookupRecords(addr=$id).nombre"]
}],
# TODO This is a symptom of comparing before and after values using rich values that refer
# to a destroyed column (a ColumnView). In reality, the values before and after after the
# same, but here the attempt to encode the previous value produces an incorrect result.
# (It's a bug, but not easy to fix and hopefully hard to run into.)
["BulkUpdateRecord", "Address", [11, 12, 13],
{"people2": [["L", "Sam"], ["L", "Bob", "Doug"], ["L", "Alice"]]
}],
]})
def test_rename_lookup_ref_attr(self):
# Renaming `Table.lookupOne(...).foo.COLUMN` and for `.lookupRecords`
self.load_sample(self.sample)
self.add_column("Address", "people", formula="People.lookupOne(addr=$id, city=$city).addr.city")
self.add_column("Address", "people2", formula="People.lookupRecords(addr=$id).addr.city")
out_actions = self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "Address", "city", "ciudad"],
["ModifyColumn", "People", "city", {"formula": "$addr.ciudad"}],
["ModifyColumn", "Address", "people", {"formula":
"People.lookupOne(addr=$id, city=$ciudad).addr.ciudad"}],
["ModifyColumn", "Address", "people2", {"formula":
"People.lookupRecords(addr=$id).addr.ciudad"}],
["BulkUpdateRecord", "_grist_Tables_column", [21, 24, 25, 26], {
"colId": ["ciudad", "city", "people", "people2"],
"formula": ["", "$addr.ciudad",
"People.lookupOne(addr=$id, city=$ciudad).addr.ciudad",
"People.lookupRecords(addr=$id).addr.ciudad"]
}]
]})
def test_rename_lookup_iter_attr(self):
# Renaming `[x.COLUMN for x in Table.lookupRecords(...)]`.
self.load_sample(self.sample)
self.add_column("Address", "people",
formula="','.join(x.addr.city for x in People.lookupRecords(addr=$id))")
self.add_column("Address", "people2",
formula="','.join([x.addr.city for x in People.lookupRecords(addr=$id)])")
self.add_column("Address", "people3",
formula="','.join({x.addr.city for x in People.lookupRecords(addr=$id)})")
self.add_column("Address", "people4",
formula="{x.addr.city:x.addr for x in People.lookupRecords(addr=$id)}")
out_actions = self.apply_user_action(["RenameColumn", "People", "addr", "ADDRESS"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "People", "addr", "ADDRESS"],
["ModifyColumn", "People", "city", {"formula": "$ADDRESS.city"}],
["ModifyColumn", "Address", "people",
{"formula": "','.join(x.ADDRESS.city for x in People.lookupRecords(ADDRESS=$id))"}],
["ModifyColumn", "Address", "people2",
{"formula": "','.join([x.ADDRESS.city for x in People.lookupRecords(ADDRESS=$id)])"}],
["ModifyColumn", "Address", "people3",
{"formula": "','.join({x.ADDRESS.city for x in People.lookupRecords(ADDRESS=$id)})"}],
["ModifyColumn", "Address", "people4",
{"formula": "{x.ADDRESS.city:x.ADDRESS for x in People.lookupRecords(ADDRESS=$id)}"}],
["BulkUpdateRecord", "_grist_Tables_column", [23, 24, 25, 26, 27, 28], {
"colId": ["ADDRESS", "city", "people", "people2", "people3", "people4"],
"formula": ["", "$ADDRESS.city",
"','.join(x.ADDRESS.city for x in People.lookupRecords(ADDRESS=$id))",
"','.join([x.ADDRESS.city for x in People.lookupRecords(ADDRESS=$id)])",
"','.join({x.ADDRESS.city for x in People.lookupRecords(ADDRESS=$id)})",
"{x.ADDRESS.city:x.ADDRESS for x in People.lookupRecords(ADDRESS=$id)}"],
}],
]})
def test_rename_table(self):
# Renaming TABLE.lookupFunc(...) where TABLE is a user-defined table.
self.load_sample(self.sample)
self.add_column("Address", "people", formula="People.lookupRecords(addr=$id)")
self.add_column("Address", "people2", type="Ref:People", formula="People.lookupOne(addr=$id)")
out_actions = self.apply_user_action(["RenameTable", "People", "Persons"])
self.assertPartialOutActions(out_actions, { "stored": [
["ModifyColumn", "Address", "people2", {"type": "Int"}],
["RenameTable", "People", "Persons"],
["UpdateRecord", "_grist_Tables", 2, {"tableId": "Persons"}],
["ModifyColumn", "Address", "people2", {
"type": "Ref:Persons", "formula": "Persons.lookupOne(addr=$id)" }],
["ModifyColumn", "Address", "people", {"formula": "Persons.lookupRecords(addr=$id)"}],
["BulkUpdateRecord", "_grist_Tables_column", [26, 25], {
"type": ["Ref:Persons", "Any"],
"formula": ["Persons.lookupOne(addr=$id)", "Persons.lookupRecords(addr=$id)"]
}],
]})
def test_rename_table_autocomplete(self):
# Renaming a table should not leave the old name available for auto-complete.
self.load_sample(self.sample)
names = {"People", "Persons"}
self.assertEqual(names.intersection(self.engine.autocomplete("Pe", "Address")), {"People"})
# Rename the table and ensure that "People" is no longer present among top-level names.
out_actions = self.apply_user_action(["RenameTable", "People", "Persons"])
self.assertEqual(names.intersection(self.engine.autocomplete("Pe", "Address")), {"Persons"})
def test_rename_to_id(self):
# Check that we renaming a column to "Id" disambiguates it with a suffix.
self.load_sample(self.sample)
out_actions = self.apply_user_action(["RenameColumn", "People", "name", "Id"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "People", "name", "Id2"],
["UpdateRecord", "_grist_Tables_column", 22, {"colId": "Id2"}],
]})
def test_renames_with_non_ascii(self):
# Test that presence of unicode does not interfere with formula adjustments for renaming.
self.load_sample(self.sample)
self.add_column("Address", "CityUpper", formula="'Øî'+$city.upper()+'áü'")
out_actions = self.apply_user_action(["RenameColumn", "Address", "city", "ciudad"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "Address", "city", "ciudad"],
["ModifyColumn", "People", "city", {"formula": "$addr.ciudad"}],
["ModifyColumn", "Address", "CityUpper", {"formula": "'Øî'+$ciudad.upper()+'áü'"}],
["BulkUpdateRecord", "_grist_Tables_column", [21, 24, 25], {
"colId": ["ciudad", "city", "CityUpper"],
"formula": ["", "$addr.ciudad", "'Øî'+$ciudad.upper()+'áü'"],
}]
]})
self.assertTableData("Address", cols="all", data=[
["id", "ciudad", "CityUpper"],
[11, "New York", "ØîNEW YORKáü"],
[12, "Colombia", "ØîCOLOMBIAáü"],
[13, "New Haven", "ØîNEW HAVENáü"],
[14, "West Haven", "ØîWEST HAVENáü"],
])
def test_rename_updates_properties(self):
# This tests for the following bug: a column A of type Any with formula Table1.lookupOne(B=$B)
# will return a correct reference; when column Table1.X is renamed to Y, $A.X will be changed
# to $A.Y correctly. The bug was that the fixed $A.Y formula would fail incorrectly with
# "Table1 has no column 'Y'".
#
# The cause was that Record objects created by $A were not affected by the
# rename, or recomputed after it, and contained a stale list of allowed column names (the fix
# removes reliance on storing column names in the Record class).
self.load_sample(self.sample)
self.add_column("Address", "person", formula="People.lookupOne(addr=$id)")
self.add_column("Address", "name", formula="$person.name")
from datetime import date
# A helper for comparing Record objects below.
people_table = self.engine.tables['People']
people_rec = lambda row_id: people_table.Record(people_table, row_id, None)
# Verify the data and calculations are correct.
self.assertTableData("Address", cols="all", data=[
["id", "city", "person", "name"],
[11, "New York", people_rec(4), "Sam"],
[12, "Colombia", people_rec(1), "Bob"],
[13, "New Haven", people_rec(2), "Alice"],
[14, "West Haven", people_rec(0), ""],
])
# Do the rename.
out_actions = self.apply_user_action(["RenameColumn", "People", "name", "name2"])
self.assertPartialOutActions(out_actions, { "stored": [
["RenameColumn", "People", "name", "name2"],
["ModifyColumn", "Address", "name", {"formula": "$person.name2"}],
["BulkUpdateRecord", "_grist_Tables_column", [22, 26], {
"colId": ["name2", "name"],
"formula": ["", "$person.name2"],
}]
]})
# Verify the data and calculations are correct after the rename.
self.assertTableData("Address", cols="all", data=[
["id", "city", "person", "name"],
[11, "New York", people_rec(4), "Sam"],
[12, "Colombia", people_rec(1), "Bob"],
[13, "New Haven", people_rec(2), "Alice"],
[14, "West Haven", people_rec(0), ""],
])