diff --git a/app/client/components/ActionLog.ts b/app/client/components/ActionLog.ts index ce89d992..6bdb8ec0 100644 --- a/app/client/components/ActionLog.ts +++ b/app/client/components/ActionLog.ts @@ -261,6 +261,7 @@ export class ActionLog extends dispose.Disposable implements IDomComponent { private async _loadActionSummaries() { if (this._loaded || !this._gristDoc) { return; } this._loading(true); + // Returned actions are ordered with earliest actions first. const result = await this._gristDoc!.docComm.getActionSummaries(); this._loading(false); this._loaded = true; @@ -268,7 +269,7 @@ export class ActionLog extends dispose.Disposable implements IDomComponent { result.forEach(item => this.pushAction(item)); // Add any actions that came in while we were fetching. Unlikely, but // perhaps possible? - const top = result[0] ? result[0].actionNum : 0; + const top = result.length > 0 ? result[result.length - 1].actionNum : 0; for (const item of this._pending) { if (item.actionNum > top) { this.pushAction(item); } } diff --git a/app/server/lib/ActionHistory.ts b/app/server/lib/ActionHistory.ts index e01ad9e8..b324648c 100644 --- a/app/server/lib/ActionHistory.ts +++ b/app/server/lib/ActionHistory.ts @@ -31,6 +31,9 @@ export interface ActionGroupOptions { // Values returned by the action, if known. retValues?: any[]; + + // Set the 'internal' flag on the created actions, as inappropriate to undo. + internal?: boolean; } export abstract class ActionHistory { @@ -259,8 +262,6 @@ export function asActionGroup(history: ActionHistory, rowIdHint, primaryAction, isUndo, - internal: act.actionNum === 0 // Mark lazy-loading calculated columns. In future, - // synchronizing fields to today's date and other - // changes from external values may count as internal. + internal: options.internal || false, }; } diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 768275ff..c1814619 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -398,7 +398,7 @@ export class ActiveDoc extends EventEmitter { await this._actionHistory.initialize(); this._granularAccess = new GranularAccess(this.docData, (query) => { - return this.fetchQuery(makeExceptionalDocSession('system'), query, true) + return this.fetchQuery(makeExceptionalDocSession('system'), query, true); }); await this._granularAccess.update(); this._sharing = new Sharing(this, this._actionHistory); @@ -581,7 +581,7 @@ export class ActiveDoc extends EventEmitter { // If user does not have rights to access what this query is asking for, fail. const tableAccess = this._granularAccess.getTableAccess(docSession, query.tableId); - if (!(tableAccess.permission & Permissions.VIEW)) { + if (!(tableAccess.permission & Permissions.VIEW)) { // tslint:disable-line:no-bitwise throw new Error('not authorized to read table'); } @@ -1042,6 +1042,7 @@ export class ActiveDoc extends EventEmitter { * isModification: true if document was changed by one or more actions. * } */ + @ActiveDoc.keepDocOpen protected async _applyUserActions(docSession: OptDocSession, actions: UserAction[], options: ApplyUAOptions = {}): Promise { @@ -1125,6 +1126,10 @@ export class ActiveDoc extends EventEmitter { * collaborators. */ private async _migrate(docSession: OptDocSession): Promise { + // TODO fetchAllTables() creates more memory pressure than usual since full data is present in + // memory at once both in node and in Python. This is liable to cause crashes. This doesn't + // even skip onDemand tables. We should try to migrate using metadata only. Data migrations + // would need to be done table-by-table, and differently still for on-demand tables. const allTables = await this.docStorage.fetchAllTables(); const docActions: DocAction[] = await this._dataEngine.pyCall('create_migrations', allTables); this.logInfo(docSession, "_migrate: applying %d migration actions", docActions.length); diff --git a/app/server/lib/DocStorage.ts b/app/server/lib/DocStorage.ts index 7356eaf3..505f04d6 100644 --- a/app/server/lib/DocStorage.ts +++ b/app/server/lib/DocStorage.ts @@ -34,6 +34,8 @@ const debuglog = util.debuglog('db'); const maxSQLiteVariables = 500; // Actually could be 999, so this is playing it safe. +const PENDING_VALUE = [gristTypes.GristObjCode.Pending]; + export class DocStorage implements ISQLiteDB { // ====================================================================== @@ -332,6 +334,40 @@ export class DocStorage implements ISQLiteDB { } } }, + + async function(db: SQLiteDB): Promise { + // Storage version 7. Migration to store formulas in SQLite. + // Here, we only create empty columns for each formula column in the document. We let + // ActiveDoc, when it calculates formulas on open, detect that this migration just + // happened, and save the calculated results. + const colRows: ResultRow[] = await db.all('SELECT t.tableId, c.colId, c.type ' + + 'FROM _grist_Tables_column c JOIN _grist_Tables t ON c.parentId=t.id WHERE c.isFormula'); + + // Go table by table. + const tableColRows = groupBy(colRows, 'tableId'); + for (const tableId of Object.keys(tableColRows)) { + // There should be no columns conflicting with formula columns, but we check and skip + // them if there are. + const infoRows = await db.all(`PRAGMA table_info(${quoteIdent(tableId)})`); + const presentCols = new Set([...infoRows.map(row => row.name)]); + const newCols = tableColRows[tableId].filter(c => !presentCols.has(c.colId)); + + // Create all new columns. + for (const {colId, type} of newCols) { + await db.exec(`ALTER TABLE ${quoteIdent(tableId)} ` + + `ADD COLUMN ${DocStorage._columnDef(colId, type)}`); + } + + // Fill them in with PENDING_VALUE. This way, on first load and Calculate, they would go + // from "Loading..." to their proper value. After the migration, they should never have + // PENDING_VALUE again. + const colListSql = newCols.map(c => `${quoteIdent(c.colId)}=?`).join(', '); + const types = newCols.map(c => c.type); + const sqlParams = DocStorage._encodeColumnsToRows(types, newCols.map(c => [PENDING_VALUE])); + await db.run(`UPDATE ${quoteIdent(tableId)} SET ${colListSql}`, sqlParams[0]); + } + }, + ] }; @@ -758,9 +794,7 @@ export class DocStorage implements ISQLiteDB { public _process_AddTable(tableId: string, columns: any[]): Promise { const colSpecSql = DocStorage._prefixJoin(', ', - columns.filter(c => - !c.isFormula).map(c => - DocStorage._columnDef(c.id, c.type))); + columns.map(c => DocStorage._columnDef(c.id, c.type))); // Every table needs an "id" column, and it should be an "integer primary key" type so that it // serves as the alias for the SQLite built-in "rowid" column. See @@ -923,10 +957,6 @@ export class DocStorage implements ISQLiteDB { * @returns {Promise} - A promise for the SQL execution. */ public async _process_AddColumn(tableId: string, colId: string, colInfo: any): Promise { - // No need to ALTER TABLE for formula columns - if (colInfo.isFormula) { - return; - } await this.exec( `ALTER TABLE ${quoteIdent(tableId)} ADD COLUMN ${DocStorage._columnDef(colId, colInfo.type)}`); } @@ -943,14 +973,8 @@ export class DocStorage implements ISQLiteDB { if (fromColId === 'id' || fromColId === 'manualSort' || tableId.startsWith('_grist')) { throw new Error('Cannot rename internal Grist column'); } - try { - await this.exec( - `ALTER TABLE ${quoteIdent(tableId)} RENAME COLUMN ${quoteIdent(fromColId)} TO ${quoteIdent(toColId)}`); - } catch (error) { - if (!String(error).match(/SQLITE_ERROR: no such column/)) { throw error; } - // Accept no-such-column, because this may be a formula column. - // TODO: tighten constraint by getting access to grist schema info and isFormula flag. - } + await this.exec( + `ALTER TABLE ${quoteIdent(tableId)} RENAME COLUMN ${quoteIdent(fromColId)} TO ${quoteIdent(toColId)}`); } /** @@ -968,20 +992,7 @@ export class DocStorage implements ISQLiteDB { log.error("ModifyColumn action called without params."); return; } - - if (colInfo.isFormula) { - return this._process_RemoveColumn(tableId, colId); - } else { - const colExists = await this._colExistsInDB(tableId, colId); - const toNonFormula = colInfo.hasOwnProperty('isFormula') && !colInfo.isFormula; - if (!colExists && toNonFormula) { - // It's important to have the original type here, when colInfo does not include it. - if (!colInfo.type) { colInfo.type = this._getGristType(tableId, colId); } - return this._process_AddColumn(tableId, colId, colInfo); - } else { - return this._alterColumn(tableId, colId, colId, colInfo.type); - } - } + return this._alterColumn(tableId, colId, colId, colInfo.type); } @@ -1301,14 +1312,6 @@ export class DocStorage implements ISQLiteDB { } } - /** - * Returns a promise for a boolean for whether the given column exists in the database. - */ - private _colExistsInDB(tableId: string, colId: string): Promise { - return this.all(`PRAGMA table_info(${quoteIdent(tableId)})`) - .then(infoRows => infoRows.some(row => (row.name === colId))); - } - private _getGristType(tableId: string, colId: string): string { return (this._docSchema[tableId] && this._docSchema[tableId][colId]) || 'Any'; } diff --git a/app/server/lib/ExpandedQuery.ts b/app/server/lib/ExpandedQuery.ts index 37ab8dae..f2ec682f 100644 --- a/app/server/lib/ExpandedQuery.ts +++ b/app/server/lib/ExpandedQuery.ts @@ -36,9 +36,9 @@ export interface ExpandedQuery extends Query { * The referenced column itself cannot (yet) be a formula. * Filtered columns cannot (yet) be a formula. * - * If formulas is not set, we simply mark formula columns as pending. + * If onDemandFormulas is set, ignore stored formula columns, and compute them using SQL. */ -export function expandQuery(iquery: Query, docData: DocData, formulas: boolean = true): ExpandedQuery { +export function expandQuery(iquery: Query, docData: DocData, onDemandFormulas: boolean = true): ExpandedQuery { const query: ExpandedQuery = { tableId: iquery.tableId, filters: iquery.filters, @@ -63,11 +63,12 @@ export function expandQuery(iquery: Query, docData: DocData, formulas: boolean = const joins = new Set(); const selects = new Set(); - // Select all data columns - selects.add(`${quoteIdent(query.tableId)}.*`); - // Iterate through all formulas, adding joins and selects as we go. - if (formulas) { + if (onDemandFormulas) { + selects.add(`${quoteIdent(query.tableId)}.id`); + for (const column of dataColumns) { + selects.add(`${quoteIdent(query.tableId)}.${quoteIdent(column.colId as string)}`); + } const formulaColumns = columns.filterRecords({parentId: tableRef, isFormula: true}); for (const column of formulaColumns) { const formula = parseFormula(column.formula as string); @@ -118,14 +119,8 @@ export function expandQuery(iquery: Query, docData: DocData, formulas: boolean = } } } else { - const formulaColumns = columns.filterRecords({parentId: tableRef, isFormula: true}); - for (const column of formulaColumns) { - if (!column.formula) { continue; } // Columns like this won't get calculated, so skip. - const colId = column.colId as string; - if (!query.constants) { query.constants = {}; } - query.constants[colId] = ['P']; - selects.add(`0 as ${quoteIdent(colId)}`); - } + // Select all data and formula columns. + selects.add(`${quoteIdent(query.tableId)}.*`); } // Copy decisions to the query object, and return. diff --git a/app/server/lib/Sharing.ts b/app/server/lib/Sharing.ts index e197f498..4d30d588 100644 --- a/app/server/lib/Sharing.ts +++ b/app/server/lib/Sharing.ts @@ -213,9 +213,10 @@ export class Sharing { // action that initializes them when the document is opened cold // (without cached ActiveDoc) - otherwise we'll end up with spam // log entries for each time the document is opened cold. - const trivial = (userActions.length === 1 && - userActions[0][0] === 'Calculate' && - sandboxActionBundle.stored.length === 0); + + const isCalculate = (userActions.length === 1 && + userActions[0][0] === 'Calculate'); + const trivial = isCalculate && sandboxActionBundle.stored.length === 0; const actionNum = trivial ? 0 : (branch === Branch.Shared ? this._actionHistory.getNextHubActionNum() : @@ -256,7 +257,9 @@ export class Sharing { // be shared. Once sharing is enabled, we would share a snapshot at that time. await this._actionHistory.recordNextShared(localActionBundle); } - if (client && client.clientId) { + // Check isCalculate because that's not an action we should allow undo/redo for (it's not + // considered as performed by a particular client). + if (client && client.clientId && !isCalculate) { this._actionHistory.setActionClientId(localActionBundle.actionHash!, client.clientId); } }); @@ -274,6 +277,9 @@ export class Sharing { client, retValues: sandboxActionBundle.retValues, summarize: true, + // Mark the on-open Calculate action as internal. In future, synchronizing fields to today's + // date and other changes from external values may count as internal. + internal: isCalculate, }); await this._activeDoc.broadcastDocUpdate(client || null, 'docUserAction', { actionGroup, diff --git a/sandbox/grist/action_obj.py b/sandbox/grist/action_obj.py index 0bff6576..212515a5 100644 --- a/sandbox/grist/action_obj.py +++ b/sandbox/grist/action_obj.py @@ -10,9 +10,8 @@ ActionGroup. In a separate step, this ActionGroup is split up according to ACL r ActionBundle consisting of ActionEnvelopes, each containing a smaller set of actions associated with the set of recipients who should receive them. """ - import actions - +from action_summary import ActionSummary class ActionGroup(object): """ @@ -26,6 +25,21 @@ class ActionGroup(object): self.stored = [] self.undo = [] self.retValues = [] + self.summary = ActionSummary() + + def flush_calc_changes(self): + """ + Merge the changes from self.summary into self.stored and self.undo, and clear the summary. + """ + self.summary.convert_deltas_to_actions(self.stored, self.undo) + self.summary = ActionSummary() + + def flush_calc_changes_for_column(self, table_id, col_id): + """ + Merge the changes for the given column from self.summary into self.stored and self.undo, and + remove that column from the summary. + """ + self.summary.pop_column_delta_as_actions(table_id, col_id, self.stored, self.undo) def get_repr(self): return { diff --git a/sandbox/grist/action_summary.py b/sandbox/grist/action_summary.py new file mode 100644 index 00000000..4a828389 --- /dev/null +++ b/sandbox/grist/action_summary.py @@ -0,0 +1,182 @@ +""" +Representation of changes due to some actions, similar to app/common/ActionSummary on node side. +It's used for collecting calculated values for formula columns. +""" +from collections import namedtuple + +import actions +from objtypes import equal_encoding + +# Pairs of before/after names of tables and columns. None represents non-existence for `before`, +# while "defunct_name" (i.e. `-{name}`) represents non-existence for `after`. This way, +# addition and removal of tables/columns can be represented. +# +# Note that changes are keyed using the last known name, or "defunct_name" for entities that have +# been removed. +LabelDelta = namedtuple('before', 'after') + +class ActionSummary(object): + # This is a class (similar to app/common/ActionSummary on node side) to summarize a list of + # docactions to easily answer questions such as whether a column was added. + def __init__(self): + self._tables = {} # maps tableId to TableDelta + self._table_renames = LabelRenames() + + def add_changes(self, table_id, col_id, changes): + """ + Record changes for the given table and column, in the form (row_id, before, after). + """ + col_deltas = self._forTable(table_id).column_deltas.setdefault(col_id, {}) + for (row_id, before, after) in changes: + # If a change was already recorded, update the 'after' value and keep the 'before' one. + previous = col_deltas.get(row_id) + col_deltas[row_id] = (previous[0] if previous else before, after) + + def convert_deltas_to_actions(self, out_stored, out_undo): + """ + Go through all prepared deltas, construct DocActions for them, and add them to out_stored + and out_undo lists. + """ + for table_id in sorted(self._tables): + table_delta = self._tables[table_id] + for col_id in sorted(table_delta.column_deltas): + column_delta = table_delta.column_deltas[col_id] + self._changes_to_actions(table_id, col_id, column_delta, out_stored, out_undo) + + def pop_column_delta_as_actions(self, table_id, col_id, out_stored, out_undo): + """ + Remove deltas for a particular column, and convert the removed deltas to DocActions. Add + those to out_stored and out_undo lists. + """ + table_delta = self._tables.get(table_id) + col_delta = table_delta and table_delta.column_deltas.pop(col_id, None) + return self._changes_to_actions(table_id, col_id, col_delta or {}, out_stored, out_undo) + + def _changes_to_actions(self, table_id, col_id, column_delta, out_stored, out_undo): + """ + Given a column and a dict of column_deltas for it, of the form {row_id: (before_value, + after_value)}, creates DocActions and adds them to out_stored and out_undo lists. + """ + if not column_delta: + return + full_row_ids = sorted(r for r, (before, after) in column_delta.iteritems() + if not equal_encoding(before, after)) + + defunct = is_defunct(table_id) or is_defunct(col_id) + table_id = root_name(table_id) + col_id = root_name(col_id) + + if not defunct: + row_ids = self.filter_out_gone_rows(table_id, full_row_ids) + if row_ids: + values = [column_delta[r][1] for r in row_ids] + out_stored.append(actions.BulkUpdateRecord(table_id, row_ids, {col_id: values}).simplify()) + + if self.is_created(table_id, col_id) and not defunct: + # A newly-create column, and not replacing a defunct one. Don't generate undo actions. + pass + else: + row_ids = self.filter_out_new_rows(table_id, full_row_ids) + if row_ids: + values = [column_delta[r][0] for r in row_ids] + undo_action = actions.BulkUpdateRecord(table_id, row_ids, {col_id: values}).simplify() + if defunct: + # If we deleted the column (or its containing table), then during undo, the updates for it + # should come after it's re-added. So we need to insert the undos *before*. + out_undo.insert(0, undo_action) + else: + out_undo.append(undo_action) + + + def _forTable(self, table_id): + return self._tables.get(table_id) or self._tables.setdefault(table_id, TableDelta()) + + def is_created(self, table_id, col_id): + if self._table_renames.is_created(table_id): + return True + t = self._tables.get(table_id) + return t and t.column_renames.is_created(col_id) + + def filter_out_new_rows(self, table_id, row_ids): + t = self._tables.get(table_id) + if not t: + return row_ids + return [r for r in row_ids if t._rows_present_before.get(r) != False] + + def filter_out_gone_rows(self, table_id, row_ids): + t = self._tables.get(table_id) + if not t: + return row_ids + return [r for r in row_ids if t._rows_present_after.get(r) != False] + + def add_records(self, table_id, row_ids): + t = self._forTable(table_id) + for r in row_ids: + # An addition means the row was initially absent, unless we already processed its removal. + t._rows_present_before.setdefault(r, False) + t._rows_present_after[r] = True + + def remove_records(self, table_id, row_ids): + t = self._forTable(table_id) + for r in row_ids: + # A removal means the row was initially present, unless it was already marked as new. + t._rows_present_before.setdefault(r, True) + t._rows_present_after[r] = False + + def add_column(self, table_id, col_id): + return self.rename_column(table_id, None, col_id) + + def remove_column(self, table_id, col_id): + return self.rename_column(table_id, col_id, defunct_name(col_id)) + + def rename_column(self, table_id, old_col_id, new_col_id): + t = self._forTable(table_id) + t.column_renames.add_rename(old_col_id, new_col_id) + if old_col_id in t.column_deltas: + t.column_deltas[new_col_id] = t.column_deltas.pop(old_col_id) + + def add_table(self, table_id): + self.rename_table(None, table_id) + + def remove_table(self, table_id): + self.rename_table(table_id, defunct_name(table_id)) + + def rename_table(self, old_table_id, new_table_id): + self._table_renames.add_rename(old_table_id, new_table_id) + if old_table_id in self._tables: + self._tables[new_table_id] = self._tables.pop(old_table_id) + +class TableDelta(object): + def __init__(self): + # Each map maps rowId to True or False. If a row was added and later removed, both will be + # False. If removed, then added, both will be True. If neither, it will not be in the map. + self._rows_present_before = {} + self._rows_present_after = {} + self.column_renames = LabelRenames() + self.column_deltas = {} # maps col_id to the dict {row_id: (before_value, after_value)} + + +class LabelRenames(object): + """ + Maintains a set of renames, for tables in a doc, or for columns in a table. For now, we only + maintain the knowledge of the original name, since we only need to answer limited questions. + """ + def __init__(self): + self._new_to_old = {} + + def add_rename(self, before, after): + original = self._new_to_old.pop(before, before) + self._new_to_old[after] = original + + def is_created(self, latest_name): + return self._new_to_old.get(latest_name, latest_name) is None + + +def defunct_name(name): + return '-' + name + +def is_defunct(name): + return name.startswith('-') + +def root_name(name): + return name[1:] if name.startswith('-') else name diff --git a/sandbox/grist/docactions.py b/sandbox/grist/docactions.py index fffb9f3d..cf72d8cd 100644 --- a/sandbox/grist/docactions.py +++ b/sandbox/grist/docactions.py @@ -1,7 +1,7 @@ import actions import schema import logger -from usertypes import strict_equal +from objtypes import strict_equal log = logger.Logger(__name__, logger.INFO) @@ -24,6 +24,7 @@ class DocActions(object): "docactions.[Bulk]AddRecord for existing record #%s" % row_id self._engine.out_actions.undo.append(actions.BulkRemoveRecord(table_id, row_ids).simplify()) + self._engine.out_actions.summary.add_records(table_id, row_ids) self._engine.add_records(table_id, row_ids, column_values) @@ -37,7 +38,7 @@ class DocActions(object): # make sure we don't have stale values hanging around. undo_values = {} for column in table.all_columns.itervalues(): - if not column.is_formula() and column.col_id != "id": + if not column.is_private() and column.col_id != "id": col_values = map(column.raw_get, row_ids) default = column.getdefault() # If this column had all default values, don't include it into the undo BulkAddRecord. @@ -49,6 +50,7 @@ class DocActions(object): # Generate the undo action. self._engine.out_actions.undo.append( actions.BulkAddRecord(table_id, row_ids, undo_values).simplify()) + self._engine.out_actions.summary.remove_records(table_id, row_ids) # Invalidate the deleted rows, so that anything that depends on them gets recomputed. self._engine.invalidate_records(table_id, row_ids) @@ -83,6 +85,8 @@ class DocActions(object): def ReplaceTableData(self, table_id, row_ids, column_values): old_data = self._engine.fetch_table(table_id, formulas=False) self._engine.out_actions.undo.append(actions.ReplaceTableData(*old_data)) + self._engine.out_actions.summary.remove_records(table_id, old_data[1]) + self._engine.out_actions.summary.add_records(table_id, row_ids) self._engine.load_table(actions.TableData(table_id, row_ids, column_values)) #---------------------------------------- @@ -100,6 +104,7 @@ class DocActions(object): # Generate the undo action. self._engine.out_actions.undo.append(actions.RemoveColumn(table_id, col_id)) + self._engine.out_actions.summary.add_column(table_id, col_id) def RemoveColumn(self, table_id, col_id): table = self._engine.tables[table_id] @@ -108,23 +113,30 @@ class DocActions(object): # Generate (if needed) the undo action to restore the data. undo_action = None column = table.get_column(col_id) - if not column.is_formula(): + if not column.is_private(): default = column.getdefault() # Add to undo a BulkUpdateRecord for non-default values in the column being removed. - row_ids = [r for r in table.row_ids if not strict_equal(column.raw_get(r), default)] - undo_action = actions.BulkUpdateRecord(table_id, row_ids, { - column.col_id: map(column.raw_get, row_ids) - }).simplify() + undo_values = [(r, column.raw_get(r)) for r in table.row_ids + if not strict_equal(column.raw_get(r), default)] # Remove the specified column from the schema object. colinfo = self._engine.schema[table_id].columns.pop(col_id) self._engine.rebuild_usercode() - # Generate the undo action(s). - if undo_action: - self._engine.out_actions.undo.append(undo_action) + # Generate the undo action(s); if for a formula column, add them to the calc summary. + if undo_values: + if column.is_formula(): + changes = [(r, v, default) for (r, v) in undo_values] + self._engine.out_actions.summary.add_changes(table_id, col_id, changes) + else: + row_ids = [r for (r, v) in undo_values] + values = [v for (r, v) in undo_values] + undo_action = actions.BulkUpdateRecord(table_id, row_ids, {col_id: values}).simplify() + self._engine.out_actions.undo.append(undo_action) + self._engine.out_actions.undo.append(actions.AddColumn( table_id, col_id, schema.col_to_dict(colinfo, include_id=False))) + self._engine.out_actions.summary.remove_column(table_id, col_id) def RenameColumn(self, table_id, old_col_id, new_col_id): table = self._engine.tables[table_id] @@ -150,6 +162,7 @@ class DocActions(object): # Generate the undo action. self._engine.out_actions.undo.append(actions.RenameColumn(table_id, new_col_id, old_col_id)) + self._engine.out_actions.summary.rename_column(table_id, old_col_id, new_col_id) def ModifyColumn(self, table_id, col_id, col_info): table = self._engine.tables[table_id] @@ -198,12 +211,13 @@ class DocActions(object): # Generate the undo action. self._engine.out_actions.undo.append(actions.RemoveTable(table_id)) + self._engine.out_actions.summary.add_table(table_id) def RemoveTable(self, table_id): assert table_id in self._engine.tables, "Table %s doesn't exist" % table_id # Create undo actions to restore all the data records of this table. - table_data = self._engine.fetch_table(table_id, formulas=False) + table_data = self._engine.fetch_table(table_id, formulas=True) undo_action = actions.BulkAddRecord(*table_data).simplify() if undo_action: self._engine.out_actions.undo.append(undo_action) @@ -215,6 +229,7 @@ class DocActions(object): # Generate the undo action. self._engine.out_actions.undo.append(actions.AddTable( table_id, schema.cols_to_dict_list(schema_table.columns))) + self._engine.out_actions.summary.remove_table(table_id) def RenameTable(self, old_table_id, new_table_id): assert old_table_id in self._engine.tables, "Table %s doesn't exist" % old_table_id @@ -230,11 +245,12 @@ class DocActions(object): # Copy over all columns from the old table to the new. new_table = self._engine.tables[new_table_id] for new_column in new_table.all_columns.itervalues(): - if not new_column.is_formula(): + if not new_column.is_private(): new_column.copy_from_column(old_table.get_column(new_column.col_id)) new_table.grow_to_max() # We need to bring formula columns to the right size too. # Generate the undo action. self._engine.out_actions.undo.append(actions.RenameTable(new_table_id, old_table_id)) + self._engine.out_actions.summary.rename_table(old_table_id, new_table_id) # end diff --git a/sandbox/grist/engine.py b/sandbox/grist/engine.py index fe03c70a..2ca20b2c 100644 --- a/sandbox/grist/engine.py +++ b/sandbox/grist/engine.py @@ -25,6 +25,7 @@ import gencode import logger import match_counter import objtypes +from objtypes import strict_equal import schema import table as table_module import useractions @@ -57,13 +58,6 @@ class OrderError(Exception): # An item of work to be done by Engine._update WorkItem = namedtuple('WorkItem', ('node', 'row_ids', 'locks')) -# Needed because some comparisons may fail (e.g. datetimes with different tzinfo objects) -def _equal_values(a, b): - try: - return a == b - except Exception: - return False - # Returns an AddTable action which can be used to reproduce the given docmodel table def _get_table_actions(table): schema_cols = [schema.make_column(c.colId, c.type, formula=c.formula, isFormula=c.isFormula) @@ -74,12 +68,6 @@ def _get_table_actions(table): # skip private members, and methods we don't want to expose to users. skipped_completions = re.compile(r'\.(_|lookupOrAddDerived|getSummarySourceGroup)') - -# Unique sentinel values with which columns are initialized before any data is loaded into them. -# For formula columns, it ensures that any calculation produces an unequal value and gets included -# into a calc action. -_pending_sentinel = object() - # The schema for the data is documented in gencode.py. # There is a general process by which values get recomputed. There are two stages: @@ -274,9 +262,9 @@ class Engine(object): for column in table.all_columns.itervalues(): column.clear() - # Only load non-formula columns + # Only load columns that aren't stored. columns = {col_id: data for (col_id, data) in data.columns.iteritems() - if table.has_column(col_id) and not table.get_column(col_id).is_formula()} + if table.has_column(col_id)} # Add the records. self.add_records(data.table_id, data.row_ids, columns) @@ -313,15 +301,6 @@ class Engine(object): for row_id, value in itertools.izip(row_ids, values): column.set(row_id, value) - # Set all values in formula columns to a special "pending" sentinel value, so that when they - # are calculated, they are considered changed and included into the produced calc actions. - # This matters because the client starts off seeing formula columns as "pending" values too. - for column in table.all_columns.itervalues(): - if not column.is_formula(): - continue - for row_id in row_ids: - column.set(row_id, _pending_sentinel) - # Invalidate new records to cause the formula columns to get recomputed. self.invalidate_records(table_id, row_ids) @@ -512,24 +491,12 @@ class Engine(object): Issues actions for any accumulated cell changes. """ for node, changes in self._changes_map.iteritems(): - if not changes: - continue table = self.tables[node.table_id] col = table.get_column(node.col_id) - # If there are changes, create and add a BulkUpdateRecord either to 'calc' or 'stored' - # actions, as appropriate. - changed_rows = [c[0] for c in changes] - changed_values = [c[1] for c in changes] - action = (actions.BulkUpdateRecord(col.table_id, changed_rows, {col.col_id: changed_values}) - .simplify()) - if action and not col.is_private(): - if col.is_formula(): - self.out_actions.calc.append(action) - else: - # We may compute values for non-formula columns (e.g. for a newly-added record), in which - # case we need a stored action. TODO: If this code path occurs during anything other than - # an AddRecord, we also need an undo action. - self.out_actions.stored.append(action) + # If there are changes, save them in out_actions. + if changes and not col.is_private(): + self.out_actions.summary.add_changes(node.table_id, node.col_id, changes) + self._pre_update() # empty lists/sets/maps def _update_loop(self, work_items, ignore_other_changes=False): @@ -656,8 +623,12 @@ class Engine(object): Public interface to recompute a column if it is dirty. It also generates a calc or stored action and adds it into self.out_actions object. """ - self._recompute_done_map.pop(col_obj.node, None) - self._recompute(col_obj.node) + self._pre_update() + try: + self._recompute_done_map.pop(col_obj.node, None) + self._recompute(col_obj.node) + finally: + self._post_update() def get_formula_error(self, table_id, col_id, row_id): """ @@ -801,10 +772,11 @@ class Engine(object): # Convert the value, and if needed, set, and include into the returned action. value = col.convert(value) - if not _equal_values(value, col.raw_get(row_id)): + previous = col.raw_get(row_id) + if not strict_equal(value, previous): if not changes: changes = self._changes_map.setdefault(node, []) - changes.append([row_id, value]) + changes.append((row_id, previous, value)) col.set(row_id, value) exclude.add(row_id) cleaned.append(row_id) @@ -1140,6 +1112,7 @@ class Engine(object): while self.docmodel.apply_auto_removes(): self._bring_all_up_to_date() + self.out_actions.flush_calc_changes() return self.out_actions def acl_split(self, action_group): @@ -1233,6 +1206,7 @@ class Engine(object): # If we changed the prefix (expanding the $ symbol) we now need to change it back. if tweaked_txt != txt: results = [txt + result[len(tweaked_txt):] for result in results] + # pylint:disable=unidiomatic-typecheck results.sort(key=lambda r: r[0] if type(r) == tuple else r) return results diff --git a/sandbox/grist/objtypes.py b/sandbox/grist/objtypes.py index 37b11dc6..c64687a7 100644 --- a/sandbox/grist/objtypes.py +++ b/sandbox/grist/objtypes.py @@ -92,6 +92,12 @@ class AltText(object): raise InvalidTypedValue(self._typename, self._text) +# Unique sentinel value representing a pending value. It's encoded as ['P'], and shown to the user +# as "Loading..." text. With the switch to stored formulas, it's currently only used when a +# document was just migrated. +_pending_sentinel = object() + + _max_js_int = 1<<31 def is_int_short(value): @@ -106,6 +112,20 @@ def safe_repr(obj): except Exception: return '<' + type(obj).__name__ + '>' +def strict_equal(a, b): + """Checks the equality of the types of the values as well as the values, and handle errors.""" + # pylint: disable=unidiomatic-typecheck + # Try/catch needed because some comparisons may fail (e.g. datetimes with different tzinfo) + try: + return type(a) == type(b) and a == b + except Exception: + return False + +def equal_encoding(a, b): + if isinstance(a, (str, unicode, float, bool, long, int)) or a is None: + # pylint: disable=unidiomatic-typecheck + return type(a) == type(b) and a == b + return encode_object(a) == encode_object(b) def encode_object(value): """ @@ -129,12 +149,17 @@ def encode_object(value): return ['d', moment.date_to_ts(value)] elif isinstance(value, RaisedException): return ['E'] + value.encode_args() - elif isinstance(value, (list, tuple, RecordList)): + elif isinstance(value, (list, tuple, RecordList, records.ColumnView)): return ['L'] + [encode_object(item) for item in value] + elif isinstance(value, records.RecordSet): + # Represent RecordSet (e.g. result of lookupRecords) in the same way as a RecordList. + return ['L'] + [encode_object(int(item)) for item in value] elif isinstance(value, dict): if not all(isinstance(key, basestring) for key in value): raise UnmarshallableError("Dict with non-string keys") return ['O', {key: encode_object(val) for key, val in value.iteritems()}] + elif value == _pending_sentinel: + return ['P'] except Exception as e: pass # We either don't know how to convert the value, or failed during the conversion. Instead we @@ -164,6 +189,8 @@ def decode_object(value): return [decode_object(item) for item in args] elif code == 'O': return {key: decode_object(val) for key, val in args[0].iteritems()} + elif code == 'P': + return _pending_sentinel raise KeyError("Unknown object type code %r" % code) except Exception as e: return RaisedException(e) diff --git a/sandbox/grist/records.py b/sandbox/grist/records.py index 97ecb3b7..08c885c5 100644 --- a/sandbox/grist/records.py +++ b/sandbox/grist/records.py @@ -144,6 +144,14 @@ class RecordSet(object): def __nonzero__(self): return bool(self._row_ids) + def __eq__(self, other): + return (isinstance(other, RecordSet) and + (self._table, self._row_ids, self._group_by, self._sort_by) == + (other._table, other._row_ids, other._group_by, other._sort_by)) + + def __ne__(self, other): + return not self.__eq__(other) + def __iter__(self): for row_id in self._row_ids: yield self.Record(self._table, row_id, self._source_relation) @@ -165,3 +173,39 @@ class RecordSet(object): relation=src_relation.compose(self._source_relation), group_by=self._group_by, sort_by=self._sort_by) + + +class ColumnView(object): + """ + ColumnView is an iterable that represents one column of a RecordSet. You may iterate through + its values and see its size, but it provides no other interface. + """ + def __init__(self, column_obj, row_ids, relation): + self._column = column_obj + self._row_ids = row_ids + self._source_relation = relation + + def __len__(self): + return len(self._row_ids) + + def __iter__(self): + for row_id in self._row_ids: + yield adjust_record(self._source_relation, self._column.get_cell_value(row_id)) + + def __eq__(self, other): + return (isinstance(other, ColumnView) and + (self._column, self._row_ids) == (other._column, other._row_ids)) + + def __ne__(self, other): + return not self.__eq__(other) + + +def adjust_record(relation, value): + """ + Helper to adjust a Record's source relation to be the composition with the given relation. This + is used to wrap values like `foo.bar`: if `bar` is a Record, then its source relation should be + the composition of the source relation of `foo` and the relation associated with `bar`. + """ + if isinstance(value, (Record, RecordSet)): + return value._clone_with_relation(relation) + return value diff --git a/sandbox/grist/runtests.py b/sandbox/grist/runtests.py index 64a8705e..7f38c075 100644 --- a/sandbox/grist/runtests.py +++ b/sandbox/grist/runtests.py @@ -8,6 +8,7 @@ tests under 'arc unit' and under Jenkins. ./sandbox/nacl/bin/run python /grist/runtests.py [--xunit] """ +import codecs import os import sys import unittest @@ -22,7 +23,8 @@ def main(): if "--xunit" in argv: import xmlrunner argv.remove("--xunit") - test_runner = xmlrunner.XMLTestRunner(stream=sys.stdout) + utf8_stdout = codecs.getwriter('utf8')(sys.stdout) + test_runner = xmlrunner.XMLTestRunner(stream=utf8_stdout) if all(arg.startswith("-") for arg in argv[1:]): argv.insert(1, "discover") diff --git a/sandbox/grist/table.py b/sandbox/grist/table.py index 13baf9f7..7bfc64a1 100644 --- a/sandbox/grist/table.py +++ b/sandbox/grist/table.py @@ -13,34 +13,6 @@ import logger log = logger.Logger(__name__, logger.INFO) -class ColumnView(object): - """ - ColumnView is an iterable that represents one column of a RecordSet. You may iterate through - its values and see its size, but it provides no other interface. - """ - def __init__(self, column_obj, row_ids, relation): - self._column = column_obj - self._row_ids = row_ids - self._source_relation = relation - - def __len__(self): - return len(self._row_ids) - - def __iter__(self): - for row_id in self._row_ids: - yield _adjust_record(self._source_relation, self._column.get_cell_value(row_id)) - - -def _adjust_record(relation, value): - """ - Helper to adjust a Record's source relation to be the composition with the given relation. This - is used to wrap values like `foo.bar`: if `bar` is a Record, then its source relation should be - the composition of the source relation of `foo` and the relation associated with `bar`. - """ - if isinstance(value, (records.Record, records.RecordSet)): - return value._clone_with_relation(relation) - return value - def _make_sample_record(table_id, col_objs): """ Helper to create a sample record for a table, used for auto-completions. @@ -468,7 +440,7 @@ class Table(object): # Called when record.foo is accessed def _get_col_value(self, col_id, row_id, relation): - return _adjust_record(relation, + return records.adjust_record(relation, self._use_column(col_id, relation, [row_id]).get_cell_value(row_id)) def _attribute_error(self, col_id, relation): @@ -479,4 +451,4 @@ class Table(object): def _get_col_subset(self, col_id, row_ids, relation): # TODO: when column is a reference, we ought to return RecordSet. Otherwise ColumnView # looks like a RecordSet (returns Records), but doesn't support property access. - return ColumnView(self._use_column(col_id, relation, row_ids), row_ids, relation) + return records.ColumnView(self._use_column(col_id, relation, row_ids), row_ids, relation) diff --git a/sandbox/grist/test_acl.py b/sandbox/grist/test_acl.py index 97aa1589..8850d99a 100644 --- a/sandbox/grist/test_acl.py +++ b/sandbox/grist/test_acl.py @@ -256,10 +256,9 @@ class TestACL(test_engine.EngineTestCase): (0, actions.UpdateRecord('Employees', 1, {'ssn': 'xxx-xx-0000'})), (1, actions.UpdateRecord('Employees', 1, {'position': 'Senior Jester'})), (0, actions.UpdateRecord('Employees', 1, {'ssn': 'yyy-yy-0000'})), + (0, actions.UpdateRecord('Employees', 1, {'salary': 100000.00})), ]) - self.assertEqual(out_bundle.calc, [ - (0, actions.UpdateRecord('Employees', 1, {'salary': 100000.00})) - ]) + self.assertEqual(out_bundle.calc, []) def test_with_rules(self): @@ -285,10 +284,9 @@ class TestACL(test_engine.EngineTestCase): self.assertEqual([(env, set(a.columns)) for (env, a) in out_bundle.stored], [ (0, {"name", "position"}), (1, {"ssn", "manualSort"}), + (1, {"salary"}), ]) - self.assertEqual([(env, set(a.columns)) for (env, a) in out_bundle.calc], [ - (1, {"salary"}) - ]) + self.assertEqual([(env, set(a.columns)) for (env, a) in out_bundle.calc], []) # Full bundle requires careful reading. See the checks above for the essential parts. self.assertEqual(out_bundle.to_json_obj(), { @@ -308,6 +306,9 @@ class TestACL(test_engine.EngineTestCase): "manualSort": [ 1, 2, 3, 4 ], "ssn": [ "000-00-0000", "111-11-1111", "222-22-2222", "222-22-2222" ] }]), + (1, [ "BulkUpdateRecord", "Employees", [ 1, 2, 3, 4 ], { + "salary": [ 60000, 100000, 60000, 100000 ] + }]), ], "undo": [ # TODO All recipients now get BulkRemoveRecord (which is correct), but some get it twice, @@ -315,11 +316,7 @@ class TestACL(test_engine.EngineTestCase): (0, [ "BulkRemoveRecord", "Employees", [ 1, 2, 3, 4 ] ]), (1, [ "BulkRemoveRecord", "Employees", [ 1, 2, 3, 4 ] ]), ], - "calc": [ - (1, [ "BulkUpdateRecord", "Employees", [ 1, 2, 3, 4 ], { - "salary": [ 60000, 100000, 60000, 100000 ] - }]) - ], + "calc": [], "retValues": [[1, 2, 3, 4]], "rules": [12,13,14], }) @@ -336,14 +333,13 @@ class TestACL(test_engine.EngineTestCase): "stored": [ (0, [ "AddRecord", "Employees", 1, {}]), (1, [ "AddRecord", "Employees", 1, {"manualSort": 1.0}]), + (1, [ "UpdateRecord", "Employees", 1, { "salary": 60000.0 }]), ], "undo": [ (0, [ "RemoveRecord", "Employees", 1 ]), (1, [ "RemoveRecord", "Employees", 1 ]), ], - "calc": [ - (1, [ "UpdateRecord", "Employees", 1, { "salary": 60000.0 }]) - ], + "calc": [], "retValues": [1], "rules": [12,13,14], }) @@ -354,13 +350,13 @@ class TestACL(test_engine.EngineTestCase): {"recipients": [ "alice", "bob", "zack" ]} ], "stored": [ (0, [ "UpdateRecord", "Employees", 1, {"position": "Senior Citizen"}]), + (1, [ "UpdateRecord", "Employees", 1, { "salary": 100000.0 }]) ], "undo": [ (0, [ "UpdateRecord", "Employees", 1, {"position": ""}]), + (1, [ "UpdateRecord", "Employees", 1, { "salary": 60000.0 }]) ], - "calc": [ - (1, [ "UpdateRecord", "Employees", 1, { "salary": 100000.0 }]) - ], + "calc": [], "retValues": [None], "rules": [12,13,14], }) diff --git a/sandbox/grist/test_derived.py b/sandbox/grist/test_derived.py index 78ff5767..01a2ce46 100644 --- a/sandbox/grist/test_derived.py +++ b/sandbox/grist/test_derived.py @@ -73,18 +73,21 @@ class TestDerived(test_engine.EngineTestCase): [2, 14] ]) self.assertPartialOutActions(out_actions, { - "calc": [actions.BulkUpdateRecord("GristSummary_6_Orders", [1,2], {'amount': [14, 29]})], + "stored": [ + actions.BulkUpdateRecord("Orders", [1,2], {'amount': [14, 14]}), + actions.BulkUpdateRecord("GristSummary_6_Orders", [1,2], {'amount': [14, 29]}) + ], "calls": {"GristSummary_6_Orders": {"amount": 2}} }) # Changing a record from one product to another should cause the two affected lines to change. out_actions = self.update_record("Orders", 10, year=2012) self.assertPartialOutActions(out_actions, { - "stored": [actions.UpdateRecord("Orders", 10, {"year": 2012})], - "calc": [ - actions.BulkUpdateRecord("GristSummary_6_Orders", [1,4], {"group": [[1,10], [7,8,9]]}), + "stored": [ + actions.UpdateRecord("Orders", 10, {"year": 2012}), actions.BulkUpdateRecord("GristSummary_6_Orders", [1,4], {"amount": [31.0, 89.0]}), actions.BulkUpdateRecord("GristSummary_6_Orders", [1,4], {"count": [2,3]}), + actions.BulkUpdateRecord("GristSummary_6_Orders", [1,4], {"group": [[1,10], [7,8,9]]}), ], "calls": {"GristSummary_6_Orders": {"group": 2, "amount": 2, "count": 2}, "Orders": {"#lookup##summary#GristSummary_6_Orders": 1, @@ -104,11 +107,9 @@ class TestDerived(test_engine.EngineTestCase): "stored": [ actions.UpdateRecord("Orders", 10, {"year": 1999}), actions.AddRecord("GristSummary_6_Orders", 5, {'year': 1999}), - ], - "calc": [ - actions.BulkUpdateRecord("GristSummary_6_Orders", [1,5], {"group": [[1], [10]]}), actions.BulkUpdateRecord("GristSummary_6_Orders", [1,5], {"amount": [14.0, 17.0]}), actions.BulkUpdateRecord("GristSummary_6_Orders", [1,5], {"count": [1,1]}), + actions.BulkUpdateRecord("GristSummary_6_Orders", [1,5], {"group": [[1], [10]]}), ], "calls": { "GristSummary_6_Orders": {'#lookup#year': 1, "group": 2, "amount": 2, "count": 2}, @@ -155,17 +156,15 @@ class TestDerived(test_engine.EngineTestCase): actions.BulkUpdateRecord("Orders", [2, 6, 7], {"product": ["B", "B", "C"]}), actions.AddRecord("GristSummary_6_Orders", 7, {'year': 2013, 'product': 'B'}), actions.AddRecord("GristSummary_6_Orders", 8, {'year': 2015, 'product': 'C'}), - ], - "calc": [ - actions.BulkUpdateRecord("GristSummary_6_Orders", [2,3,4,5,7,8], { - "group": [[3], [4,5,6], [], [10], [2], [7]] - }), actions.BulkUpdateRecord("GristSummary_6_Orders", [2,3,4,5,7,8], { "amount": [15.0, 86.0, 0, 17.0, 15.0, 17.0] }), actions.BulkUpdateRecord("GristSummary_6_Orders", [2,3,4,5,7,8], { "count": [1, 3, 0, 1, 1, 1] }), + actions.BulkUpdateRecord("GristSummary_6_Orders", [2,3,4,5,7,8], { + "group": [[3], [4,5,6], [], [10], [2], [7]] + }), ], }) @@ -216,7 +215,10 @@ class TestDerived(test_engine.EngineTestCase): # In either case, changing an amount (from 36->37 for a CT customer) should update summaries. out_actions = self.update_record('Orders', 9, amount=37) self.assertPartialOutActions(out_actions, { - "calc": [actions.UpdateRecord("GristSummary_9_Customers", 2, {"totalAmount": 135.0})] + "stored": [ + actions.UpdateRecord("Orders", 9, {"amount": 37}), + actions.UpdateRecord("GristSummary_9_Customers", 2, {"totalAmount": 135.0}), + ] }) # In either case, changing a customer's state should trigger recomputation too. @@ -286,8 +288,13 @@ class TestDerived(test_engine.EngineTestCase): [4, 2015, 4, 106.0, [7,8,9,10]], ]) self.assertPartialOutActions(out_actions_undo, { - "stored": [actions.RemoveRecord("GristSummary_6_Orders", 5), - actions.UpdateRecord("Orders", 1, {"year": 2012})], + "stored": [ + actions.UpdateRecord("GristSummary_6_Orders", 1, {"group": [1]}), + actions.UpdateRecord("GristSummary_6_Orders", 1, {"count": 1}), + actions.UpdateRecord("GristSummary_6_Orders", 1, {"amount": 15.0}), + actions.RemoveRecord("GristSummary_6_Orders", 5), + actions.UpdateRecord("Orders", 1, {"year": 2012}), + ], "calls": {"GristSummary_6_Orders": {"group": 1, "amount": 1, "count": 1}, "Orders": {"#lookup##summary#GristSummary_6_Orders": 2, "#summary#GristSummary_6_Orders": 2}} diff --git a/sandbox/grist/test_display_cols.py b/sandbox/grist/test_display_cols.py index 15465e87..b8bde40f 100644 --- a/sandbox/grist/test_display_cols.py +++ b/sandbox/grist/test_display_cols.py @@ -291,6 +291,114 @@ class TestUserActions(test_engine.EngineTestCase): [14, "Fox" , 300 ], ]) + def test_display_col_and_field_removal(self): + # When there are different displayCols associated with the column and with the field, removal + # takes more steps, and order of produced actions matters. + self.load_sample(self.ref_sample) + + # Add a table for people, which includes an associated view. + self.apply_user_action(['AddTable', 'People', [ + {'id': 'name', 'type': 'Text'}, + {'id': 'favorite', 'type': 'Ref:Television', + 'widgetOptions': '\"{\"alignment\":\"center\",\"visibleCol\":\"show\"}\"'}, + ]]) + self.apply_user_action(['BulkAddRecord', 'People', [1,2,3], { + 'name': ['Bob', 'Jim', 'Don'], + 'favorite': [12, 11, 13] + }]) + + # Add a display formula for the 'favorite' column. A "gristHelper_Display" column with the + # requested formula should be added and set as the displayCol of the favorite column. + self.apply_user_action(['SetDisplayFormula', 'People', None, 26, '$favorite.show']) + + # Set display formula for 'favorite' column field. + # A single "gristHelper_Display2" column should be added with the requested formula. + self.apply_user_action(['SetDisplayFormula', 'People', 2, None, '$favorite.network']) + + expected_tables1 = [ + Table(1, "Television", 0, 0, columns=[ + Column(21, "show", "Text", False, "", 0), + Column(22, "network", "Text", False, "", 0), + Column(23, "viewers", "Int", False, "", 0), + ]), + Table(2, "People", 1, 0, columns=[ + Column(24, "manualSort", "ManualSortPos", False, "", 0), + Column(25, "name", "Text", False, "", 0), + Column(26, "favorite", "Ref:Television", False, "", 0), + Column(27, "gristHelper_Display", "Any", True, "$favorite.show", 0), + Column(28, "gristHelper_Display2", "Any", True, "$favorite.network", 0) + ]), + ] + expected_data1 = [ + ["id", "name", "favorite", "gristHelper_Display", "gristHelper_Display2"], + [1, "Bob", 12, "Narcos", "Netflix"], + [2, "Jim", 11, "Game of Thrones", "HBO"], + [3, "Don", 13, "Today", "NBC"] + ] + self.assertTables(expected_tables1) + self.assertTableData("People", cols="subset", data=expected_data1) + self.assertTableData("_grist_Views_section_field", cols="subset", data=[ + ["id", "parentId", "colRef", "displayCol"], + [1, 1, 25, 0], + [2, 1, 26, 28], + ]) + + # Now remove the 'favorite' column. + out_actions = self.apply_user_action(['RemoveColumn', 'People', 'favorite']) + + # The associated field and both displayCols should be gone. + self.assertTables([ + expected_tables1[0], + Table(2, "People", 1, 0, columns=[ + Column(24, "manualSort", "ManualSortPos", False, "", 0), + Column(25, "name", "Text", False, "", 0), + ]), + ]) + self.assertTableData("_grist_Views_section_field", cols="subset", data=[ + ["id", "parentId", "colRef", "displayCol"], + [1, 1, 25, 0], + ]) + + # Verify that the resulting actions don't include any extraneous calc actions. + # pylint:disable=line-too-long + self.assertOutActions(out_actions, { + "stored": [ + ["RemoveRecord", "_grist_Views_section_field", 2], + ["BulkRemoveRecord", "_grist_Tables_column", [26, 27]], + ["RemoveColumn", "People", "favorite"], + ["RemoveColumn", "People", "gristHelper_Display"], + ["RemoveRecord", "_grist_Tables_column", 28], + ["RemoveColumn", "People", "gristHelper_Display2"], + ], + "undo": [ + ["BulkUpdateRecord", "People", [1, 2, 3], {"gristHelper_Display2": ["Netflix", "HBO", "NBC"]}], + ["BulkUpdateRecord", "People", [1, 2, 3], {"gristHelper_Display": ["Narcos", "Game of Thrones", "Today"]}], + ["AddRecord", "_grist_Views_section_field", 2, {"colRef": 26, "displayCol": 28, "parentId": 1, "parentPos": 2.0}], + ["BulkAddRecord", "_grist_Tables_column", [26, 27], {"colId": ["favorite", "gristHelper_Display"], "displayCol": [27, 0], "formula": ["", "$favorite.show"], "isFormula": [False, True], "label": ["favorite", "gristHelper_Display"], "parentId": [2, 2], "parentPos": [6.0, 7.0], "type": ["Ref:Television", "Any"], "widgetOptions": ["\"{\"alignment\":\"center\",\"visibleCol\":\"show\"}\"", ""]}], + ["BulkUpdateRecord", "People", [1, 2, 3], {"favorite": [12, 11, 13]}], + ["AddColumn", "People", "favorite", {"formula": "", "isFormula": False, "type": "Ref:Television"}], + ["AddColumn", "People", "gristHelper_Display", {"formula": "$favorite.show", "isFormula": True, "type": "Any"}], + ["AddRecord", "_grist_Tables_column", 28, {"colId": "gristHelper_Display2", "formula": "$favorite.network", "isFormula": True, "label": "gristHelper_Display2", "parentId": 2, "parentPos": 8.0, "type": "Any"}], + ["AddColumn", "People", "gristHelper_Display2", {"formula": "$favorite.network", "isFormula": True, "type": "Any"}], + ], + }) + + # Now undo; expect the structure and values restored. + stored_actions = out_actions.get_repr()["stored"] + undo_actions = out_actions.get_repr()["undo"] + out_actions = self.apply_user_action(['ApplyUndoActions', undo_actions]) + self.assertTables(expected_tables1) + self.assertTableData("People", cols="subset", data=expected_data1) + self.assertTableData("_grist_Views_section_field", cols="subset", data=[ + ["id", "parentId", "colRef", "displayCol"], + [1, 1, 25, 0], + [2, 1, 26, 28], + ]) + + self.assertPartialOutActions(out_actions, { + "stored": reversed(undo_actions), + }) + def test_display_col_copying(self): # Test that when switching types and using CopyFromColumn, displayCol is set/unset correctly. diff --git a/sandbox/grist/test_engine.py b/sandbox/grist/test_engine.py index b93ed6d0..7f39492f 100644 --- a/sandbox/grist/test_engine.py +++ b/sandbox/grist/test_engine.py @@ -71,7 +71,6 @@ class EngineTestCase(unittest.TestCase): """ sort_keys = {c: i for i, c in enumerate(col_names)} ret = [] - engine_data = actions.encode_objects(engine_data) for table_id, table_data in sorted(engine_data.items()): ret.append("TABLE %s\n" % table_id) col_items = sorted(table_data.columns.items(), @@ -86,9 +85,11 @@ class EngineTestCase(unittest.TestCase): Compare full engine data, as a mapping of table_ids to TableData objects, and reporting differences with a customized diff (similar to the JSON representation in the test script). """ - if observed != expected: - o_lines = self._getEngineDataLines(observed, col_names) - e_lines = self._getEngineDataLines(expected, col_names) + enc_observed = actions.encode_objects(observed) + enc_expected = actions.encode_objects(expected) + if enc_observed != enc_expected: + o_lines = self._getEngineDataLines(enc_observed, col_names) + e_lines = self._getEngineDataLines(enc_expected, col_names) self.fail("Observed data not as expected:\n" + "".join(difflib.unified_diff(e_lines, o_lines, fromfile="expected", tofile="observed"))) @@ -133,7 +134,7 @@ class EngineTestCase(unittest.TestCase): for (k, action_list) in sorted(action_group.items()): if k in cls.action_group_action_fields: for a in action_list: - rep = repr(a) if use_repr else json.dumps(actions.get_action_repr(a), sort_keys=True) + rep = repr(a) if use_repr else json.dumps(a, sort_keys=True) lines.append("%s: %s," % (k, rep)) else: lines.append("%s: %s," % (k, json.dumps(action_list))) @@ -150,19 +151,16 @@ class EngineTestCase(unittest.TestCase): # Convert expected actions into a comparable form. for k in self.action_group_action_fields: + if k in observed: + observed[k] = map(actions.get_action_repr, observed[k]) if k in expected: - expected[k] = [actions.action_from_repr(a) if isinstance(a, list) else a + expected[k] = [actions.get_action_repr(a) if not isinstance(a, list) else a for a in expected[k]] if observed != expected: o_lines = self._formatActionGroup(observed) e_lines = self._formatActionGroup(expected) - extra = "" - if o_lines == e_lines: - o_lines = self._formatActionGroup(observed, use_repr=True) - e_lines = self._formatActionGroup(expected, use_repr=True) - extra = " (BUT HAVE SAME REPR!)" - self.fail(("Observed out actions not as expected%s:\n" % extra) + + self.fail(("Observed out actions not as expected:\n") + "\n".join(difflib.unified_diff(e_lines, o_lines, n=3, lineterm="", fromfile="expected", tofile="observed"))) @@ -171,6 +169,10 @@ class EngineTestCase(unittest.TestCase): Compares action group returned from engine.apply_user_actions() to expected actions as listed in testscript. The array of retValues is only checked if present in expected_group. """ + for k in self.action_group_action_fields: + # For comparing full actions, treat omitted groups (e.g. "calc") as expected to be empty. + expected_group.setdefault(k, []) + observed = {k: getattr(out_action_group, k) for k in self.action_group_action_fields } if "retValue" in expected_group: observed["retValue"] = out_action_group.retValues @@ -190,6 +192,7 @@ class EngineTestCase(unittest.TestCase): """ output = {t: self.engine.fetch_table(t) for t in self.engine.schema} output = testutil.replace_nans(output) + output = actions.encode_objects(output) print ''.join(self._getEngineDataLines(output)) def dump_actions(self, out_actions): diff --git a/sandbox/grist/test_formula_error.py b/sandbox/grist/test_formula_error.py index 7b1cfede..9bda388a 100644 --- a/sandbox/grist/test_formula_error.py +++ b/sandbox/grist/test_formula_error.py @@ -1,8 +1,8 @@ """ Tests that formula error messages (traceback) are correct """ -import depend import textwrap +import depend import test_engine import testutil import objtypes @@ -219,17 +219,25 @@ else: # Make sure the we have bulk updates for both T and D, and not just D. err = ["E", "AttributeError"] - self.assertPartialOutActions(out_actions, { "calc": [ - [ - "BulkUpdateRecord", "UpdateTest", [1, 2, 3], { - "T": [err, err, err] - } + self.assertPartialOutActions(out_actions, { "stored": [ + ["RenameColumn", "UpdateTest", "A", "AA"], + ["ModifyColumn", "UpdateTest", "T", { + "formula": "recs = UpdateTest.lookupRecords()\nsum(r.A for r in recs if r.A <= $AA)"} + ], + ["BulkUpdateRecord", "_grist_Tables_column", [20, 21], { + "colId": ["AA", "T"], + "formula": ["", "recs = UpdateTest.lookupRecords()\nsum(r.A for r in recs if r.A <= $AA)"]} ], [ "BulkUpdateRecord", "UpdateTest", [1, 2, 3], { "D": [err, err, err] } - ] + ], + [ + "BulkUpdateRecord", "UpdateTest", [1, 2, 3], { + "T": [err, err, err] + } + ], ]}) # Make sure the table is in the correct state. @@ -515,7 +523,7 @@ else: }) self.load_sample(sample) - principal = 20213227788876 + principal = 20213227788876.0 self.assertTableData('Readout', data=[ ['id', 'LastPrincipal', 'LastRPrincipal', 'FirstTotal', 'LastTotal'], [1, principal, principal, principal + 1000, principal + 1000], diff --git a/sandbox/grist/test_formula_undo.py b/sandbox/grist/test_formula_undo.py new file mode 100644 index 00000000..c0943980 --- /dev/null +++ b/sandbox/grist/test_formula_undo.py @@ -0,0 +1,155 @@ +# pylint: disable=line-too-long +import testsamples +import test_engine + +class TestFormulaUndo(test_engine.EngineTestCase): + def setUp(self): + super(TestFormulaUndo, self).setUp() + + def test_change_and_undo(self): + self.load_sample(testsamples.sample_students) + + # Test that regular lookup results behave well on undo. + self.apply_user_action(['ModifyColumn', 'Students', 'schoolCities', { + "type": "Any", + "formula": "Schools.lookupRecords(name=$schoolName)" + }]) + + # Add a formula that produces different results on different invocations. This is + # similar to some realistic scenarious (such as returning a time, a rich python object, or a + # string like ""), but is convoluted to keep values deterministic. + self.apply_user_action(['AddColumn', 'Students', 'counter', { + "formula": """ +table.my_counter = getattr(table, 'my_counter', 0) + 1 +return '#%s %s' % (table.my_counter, $schoolName) +""" + }]) + + self.assertTableData("Students", cols="subset", data=[ + ["id", "schoolName", "schoolCities", "counter" ], + [1, "Columbia", [1, 2], "#1 Columbia",], + [2, "Yale", [3, 4], "#2 Yale", ], + [3, "Columbia", [1, 2], "#3 Columbia",], + [4, "Yale", [3, 4], "#4 Yale", ], + [5, "Eureka", [], "#5 Eureka", ], + [6, "Yale", [3, 4], "#6 Yale", ], + ]) + + # Applying an action produces expected changes to all formula columns, and corresponding undos. + out_actions = self.apply_user_action(['UpdateRecord', 'Students', 6, {"schoolName": "Columbia"}]) + self.assertOutActions(out_actions, { + "stored": [ + ["UpdateRecord", "Students", 6, {"schoolName": "Columbia"}], + ["UpdateRecord", "Students", 6, {"counter": "#7 Columbia"}], + ["UpdateRecord", "Students", 6, {"schoolCities": ["L", 1, 2]}], + ["UpdateRecord", "Students", 6, {"schoolIds": "1:2"}], + ], + "undo": [ + ["UpdateRecord", "Students", 6, {"schoolName": "Yale"}], + ["UpdateRecord", "Students", 6, {"counter": "#6 Yale"}], + ["UpdateRecord", "Students", 6, {"schoolCities": ["L", 3, 4]}], + ["UpdateRecord", "Students", 6, {"schoolIds": "3:4"}], + ], + }) + + # Applying the undo actions (which include calculated values) will trigger recalculations, but + # they should not produce extraneous actions even when the calculation results differ. + out_actions = self.apply_user_action(['ApplyUndoActions', out_actions.get_repr()["undo"]]) + + # TODO Note the double update when applying undo to non-deterministic formula. It would be + # nice to fix, but requires further refactoring (perhaps moving towards processing actions + # using summaries). + self.assertOutActions(out_actions, { + "stored": [ + ["UpdateRecord", "Students", 6, {"schoolIds": "3:4"}], + ["UpdateRecord", "Students", 6, {"schoolCities": ["L", 3, 4]}], + ["UpdateRecord", "Students", 6, {"counter": "#6 Yale"}], + ["UpdateRecord", "Students", 6, {"schoolName": "Yale"}], + ["UpdateRecord", "Students", 6, {"counter": "#8 Yale"}], + ], + "undo": [ + ["UpdateRecord", "Students", 6, {"schoolIds": "1:2"}], + ["UpdateRecord", "Students", 6, {"schoolCities": ["L", 1, 2]}], + ["UpdateRecord", "Students", 6, {"counter": "#7 Columbia"}], + ["UpdateRecord", "Students", 6, {"schoolName": "Columbia"}], + ["UpdateRecord", "Students", 6, {"counter": "#6 Yale"}], + ], + }) + + self.assertTableData("Students", cols="subset", data=[ + ["id", "schoolName", "schoolCities", "counter" ], + [1, "Columbia", [1, 2], "#1 Columbia",], + [2, "Yale", [3, 4], "#2 Yale", ], + [3, "Columbia", [1, 2], "#3 Columbia",], + [4, "Yale", [3, 4], "#4 Yale", ], + [5, "Eureka", [], "#5 Eureka", ], + [6, "Yale", [3, 4], "#8 Yale", ], # This counter got updated + ]) + + def test_save_to_empty_column(self): + # When we enter data into an empty column, it gets turned from a formula into a data column. + # Check that this operation works. + self.load_sample(testsamples.sample_students) + self.apply_user_action(['AddColumn', 'Students', 'newCol', {"isFormula": True}]) + + out_actions = self.apply_user_action(['UpdateRecord', 'Students', 6, {"newCol": "Boo!"}]) + self.assertTableData("Students", cols="subset", data=[ + ["id", "schoolName", "newCol" ], + [1, "Columbia", "" ], + [2, "Yale", "" ], + [3, "Columbia", "" ], + [4, "Yale", "" ], + [5, "Eureka", "" ], + [6, "Yale", "Boo!" ], + ]) + + # Check that the actions look reasonable. + self.assertOutActions(out_actions, { + "stored": [ + ["ModifyColumn", "Students", "newCol", {"type": "Text"}], + ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Text"}], + ["ModifyColumn", "Students", "newCol", {"isFormula": False}], + ["BulkUpdateRecord", "Students", [1,2,3,4,5,6], {"newCol": ["", "", "", "", "", ""]}], + ["UpdateRecord", "_grist_Tables_column", 22, {"isFormula": False}], + ["UpdateRecord", "Students", 6, {"newCol": "Boo!"}], + ], + "undo": [ + ["ModifyColumn", "Students", "newCol", {"type": "Any"}], + ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Any"}], + ["BulkUpdateRecord", "Students", [1,2,3,4,5,6], {"newCol": [None, None, None, None, None, None]}], + ["ModifyColumn", "Students", "newCol", {"isFormula": True}], + ["UpdateRecord", "_grist_Tables_column", 22, {"isFormula": True}], + ["UpdateRecord", "Students", 6, {"newCol": ""}], + ] + }) + + out_actions = self.apply_user_action(['ApplyUndoActions', out_actions.get_repr()["undo"]]) + self.assertTableData("Students", cols="subset", data=[ + ["id", "schoolName", "newCol" ], + [1, "Columbia", None ], + [2, "Yale", None ], + [3, "Columbia", None ], + [4, "Yale", None ], + [5, "Eureka", None ], + [6, "Yale", None ], + ]) + + # Check that undo actions are a reversal of the above, without any surprises. + self.assertOutActions(out_actions, { + "stored": [ + ["UpdateRecord", "Students", 6, {"newCol": ""}], + ["UpdateRecord", "_grist_Tables_column", 22, {"isFormula": True}], + ["ModifyColumn", "Students", "newCol", {"isFormula": True}], + ["BulkUpdateRecord", "Students", [1,2,3,4,5,6], {"newCol": [None, None, None, None, None, None]}], + ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Any"}], + ["ModifyColumn", "Students", "newCol", {"type": "Any"}], + ], + "undo": [ + ["UpdateRecord", "Students", 6, {"newCol": "Boo!"}], + ["UpdateRecord", "_grist_Tables_column", 22, {"isFormula": False}], + ["ModifyColumn", "Students", "newCol", {"isFormula": False}], + ["BulkUpdateRecord", "Students", [1,2,3,4,5,6], {"newCol": ["", "", "", "", "", ""]}], + ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Text"}], + ["ModifyColumn", "Students", "newCol", {"type": "Text"}], + ] + }) diff --git a/sandbox/grist/test_import_actions.py b/sandbox/grist/test_import_actions.py index 7c78ad88..1514467d 100644 --- a/sandbox/grist/test_import_actions.py +++ b/sandbox/grist/test_import_actions.py @@ -117,6 +117,35 @@ class TestImportActions(test_engine.EngineTestCase): [4, 1, [7]] # new section for transform preview ]) + + def test_regenereate_importer_view(self): + # Generate without a destination table, and then with one. Ensure that we don't omit the + # actions needed to populate the table in the second call. + self.init_state() + self.apply_user_action(['GenImporterView', 'Source', None, None]) + out_actions = self.apply_user_action(['GenImporterView', 'Source', 'Destination1', None]) + self.assertPartialOutActions(out_actions, { + "stored": [ + ["BulkRemoveRecord", "_grist_Views_section_field", [7, 8, 9]], + ["RemoveRecord", "_grist_Views_section", 4], + ["BulkRemoveRecord", "_grist_Tables_column", [10, 11, 12]], + ["RemoveColumn", "Source", "gristHelper_Import_Name"], + ["RemoveColumn", "Source", "gristHelper_Import_City"], + ["RemoveColumn", "Source", "gristHelper_Import_Zip"], + ["AddColumn", "Source", "gristHelper_Import_Name", {"formula": "$Name", "isFormula": True, "type": "Text"}], + ["AddRecord", "_grist_Tables_column", 10, {"colId": "gristHelper_Import_Name", "formula": "$Name", "isFormula": True, "label": "Name", "parentId": 1, "parentPos": 10.0, "type": "Text", "widgetOptions": ""}], + ["AddColumn", "Source", "gristHelper_Import_City", {"formula": "$City", "isFormula": True, "type": "Text"}], + ["AddRecord", "_grist_Tables_column", 11, {"colId": "gristHelper_Import_City", "formula": "$City", "isFormula": True, "label": "City", "parentId": 1, "parentPos": 11.0, "type": "Text", "widgetOptions": ""}], + ["AddRecord", "_grist_Views_section", 4, {"borderWidth": 1, "defaultWidth": 100, "parentKey": "record", "sortColRefs": "[]", "tableRef": 1}], + ["BulkAddRecord", "_grist_Views_section_field", [7, 8], {"colRef": [10, 11], "parentId": [4, 4], "parentPos": [7.0, 8.0]}], + # The actions to populate the removed and re-added columns should be there. + ["BulkUpdateRecord", "Source", [1, 2], {"gristHelper_Import_City": ["New York", "Boston"]}], + ["BulkUpdateRecord", "Source", [1, 2], {"gristHelper_Import_Name": ["John", "Alison"]}], + ], + "calc": [] + }) + + def test_transform_destination_new_table(self): # Add source and destination tables self.init_state() diff --git a/sandbox/grist/test_import_transform.py b/sandbox/grist/test_import_transform.py index 046e4f2e..37a4cca1 100644 --- a/sandbox/grist/test_import_transform.py +++ b/sandbox/grist/test_import_transform.py @@ -90,7 +90,33 @@ class TestImportTransform(test_engine.EngineTestCase): self.init_state() #into_new_table = True, transform_rule : no colids (will be generated for new table) - self.apply_user_action(['TransformAndFinishImport', 'Hidden_table', 'NewTable', True, self.TEMP_transform_rule_no_colids]) + out_actions = self.apply_user_action( + ['TransformAndFinishImport', 'Hidden_table', 'NewTable', True, self.TEMP_transform_rule_no_colids]) + self.assertPartialOutActions(out_actions, { + "stored": [ + ["AddColumn", "Hidden_table", "gristHelper_Import_Middle_Initial", {"formula": "$mname[0]", "isFormula": True, "type": "Text"}], + ["AddRecord", "_grist_Tables_column", 9, {"colId": "gristHelper_Import_Middle_Initial", "formula": "$mname[0]", "isFormula": True, "label": "Middle Initial", "parentId": 1, "parentPos": 9.0, "type": "Text", "widgetOptions": ""}], + ["BulkRemoveRecord", "_grist_Views_section_field", [1, 2, 3]], + ["RemoveRecord", "_grist_Views_section", 1], + ["RemoveRecord", "_grist_TabBar", 1], + ["RemoveRecord", "_grist_Pages", 1], + ["RemoveRecord", "_grist_Views", 1], + ["UpdateRecord", "_grist_Tables", 1, {"primaryViewId": 0}], + ["BulkRemoveRecord", "_grist_Tables_column", [1, 2, 3, 4, 9]], + ["RemoveRecord", "_grist_Tables", 1], + ["RemoveTable", "Hidden_table"], + ["AddTable", "NewTable", [{"formula": "", "id": "manualSort", "isFormula": False, "type": "ManualSortPos"}, {"formula": "", "id": "First_Name", "isFormula": False, "type": "Text"}, {"formula": "", "id": "Last_Name", "isFormula": False, "type": "Text"}, {"formula": "", "id": "Middle_Initial", "isFormula": False, "type": "Text"}, {"formula": "", "id": "Blank", "isFormula": False, "type": "Text"}]], + ["AddRecord", "_grist_Tables", 3, {"primaryViewId": 0, "tableId": "NewTable"}], + ["BulkAddRecord", "_grist_Tables_column", [9, 10, 11, 12, 13], {"colId": ["manualSort", "First_Name", "Last_Name", "Middle_Initial", "Blank"], "formula": ["", "", "", "", ""], "isFormula": [False, False, False, False, False], "label": ["manualSort", "First Name", "Last Name", "Middle Initial", "Blank"], "parentId": [3, 3, 3, 3, 3], "parentPos": [9.0, 10.0, 11.0, 12.0, 13.0], "type": ["ManualSortPos", "Text", "Text", "Text", "Text"], "widgetOptions": ["", "", "", "", ""]}], + ["AddRecord", "_grist_Views", 3, {"name": "NewTable", "type": "raw_data"}], + ["AddRecord", "_grist_TabBar", 3, {"tabPos": 3.0, "viewRef": 3}], + ["AddRecord", "_grist_Pages", 3, {"indentation": 0, "pagePos": 3.0, "viewRef": 3}], + ["AddRecord", "_grist_Views_section", 3, {"borderWidth": 1, "defaultWidth": 100, "parentId": 3, "parentKey": "record", "sortColRefs": "[]", "tableRef": 3, "title": ""}], + ["BulkAddRecord", "_grist_Views_section_field", [7, 8, 9, 10], {"colRef": [10, 11, 12, 13], "parentId": [3, 3, 3, 3], "parentPos": [7.0, 8.0, 9.0, 10.0]}], + ["UpdateRecord", "_grist_Tables", 3, {"primaryViewId": 3}], + ["BulkAddRecord", "NewTable", [1, 2], {"First_Name": ["Carry", "Don"], "Last_Name": ["Jonson", "Yoon"], "Middle_Initial": ["M", "B"], "manualSort": [1.0, 2.0]}], + ] + }) #1-4 in hidden table, 5-8 in destTable, 9-13 for new table self.assertTableData('_grist_Tables_column', cols="subset", data=[ diff --git a/sandbox/grist/test_lookups.py b/sandbox/grist/test_lookups.py index 60cea83b..ebcc2115 100644 --- a/sandbox/grist/test_lookups.py +++ b/sandbox/grist/test_lookups.py @@ -35,21 +35,27 @@ class TestLookups(test_engine.EngineTestCase): out_actions = self.update_record("Address", 14, city="Bedford") self.assertPartialOutActions(out_actions, { - "calc": [_bulk_update("Students", ["id", "schoolCities" ], [ - [2, "New Haven:Bedford" ], - [4, "New Haven:Bedford" ], - [6, "New Haven:Bedford" ]] - )], + "stored": [ + actions.UpdateRecord("Address", 14, {"city": "Bedford"}), + _bulk_update("Students", ["id", "schoolCities" ], [ + [2, "New Haven:Bedford" ], + [4, "New Haven:Bedford" ], + [6, "New Haven:Bedford" ]] + ) + ], "calls": {"Students": {"schoolCities": 3}} }) out_actions = self.update_record("Schools", 4, address=13) self.assertPartialOutActions(out_actions, { - "calc": [_bulk_update("Students", ["id", "schoolCities" ], [ - [2, "New Haven:New Haven" ], - [4, "New Haven:New Haven" ], - [6, "New Haven:New Haven" ]] - )], + "stored": [ + actions.UpdateRecord("Schools", 4, {"address": 13}), + _bulk_update("Students", ["id", "schoolCities" ], [ + [2, "New Haven:New Haven" ], + [4, "New Haven:New Haven" ], + [6, "New Haven:New Haven" ]] + ) + ], "calls": {"Students": {"schoolCities": 3}} }) @@ -103,7 +109,8 @@ class TestLookups(test_engine.EngineTestCase): out_actions = self.update_record("Schools", 2, name="Eureka") self.assertPartialOutActions(out_actions, { - "calc": [ + "stored": [ + actions.UpdateRecord("Schools", 2, {"name": "Eureka"}), actions.BulkUpdateRecord("Students", [1,3,5], { 'schoolCities': ["New York", "New York", "Colombia"] }), @@ -121,7 +128,8 @@ class TestLookups(test_engine.EngineTestCase): [5, "Yale"] ]) self.assertPartialOutActions(out_actions, { - "calc": [ + "stored": [ + actions.BulkUpdateRecord("Students", [3,5], {'schoolName': ["", "Yale"]}), actions.BulkUpdateRecord("Students", [3,5], {'schoolCities': ["", "New Haven:West Haven"]}), actions.BulkUpdateRecord("Students", [3,5], {'schoolIds': ["", "3:4"]}), ], @@ -148,11 +156,14 @@ class TestLookups(test_engine.EngineTestCase): # We should NOT get attribute errors in the values. out_actions = self.update_record("Schools", 4, address=13) self.assertPartialOutActions(out_actions, { - "calc": [_bulk_update("Students", ["id", "schoolCities" ], [ - [2, "New Haven:New Haven" ], - [4, "New Haven:New Haven" ], - [6, "New Haven:New Haven" ]] - )], + "stored": [ + actions.UpdateRecord("Schools", 4, {"address": 13}), + _bulk_update("Students", ["id", "schoolCities" ], [ + [2, "New Haven:New Haven" ], + [4, "New Haven:New Haven" ], + [6, "New Haven:New Haven" ]] + ) + ], "calls": { "Students": { 'schoolCities': 3 } } }) @@ -173,13 +184,21 @@ class TestLookups(test_engine.EngineTestCase): out_actions = self.modify_column("Students", "schoolCities", formula=( "','.join(Schools.lookupRecords(name=$schoolName).state)")) self.assertPartialOutActions(out_actions, { - "calc": [_bulk_update("Students", ["id", "schoolCities" ], [ - [1, "NY,MO" ], - [2, "CT,CT" ], - [3, "NY,MO" ], - [4, "CT,CT" ], - [6, "CT,CT" ]] - )], + "stored": [ + actions.ModifyColumn("Students", "schoolCities", { + "formula": "','.join(Schools.lookupRecords(name=$schoolName).state)", + }), + actions.UpdateRecord("_grist_Tables_column", 6, { + "formula": "','.join(Schools.lookupRecords(name=$schoolName).state)", + }), + _bulk_update("Students", ["id", "schoolCities" ], [ + [1, "NY,MO" ], + [2, "CT,CT" ], + [3, "NY,MO" ], + [4, "CT,CT" ], + [6, "CT,CT" ]] + ) + ], # Note that it got computed 6 times (once for each record), but one value remained unchanged # (because no schools matched). "calls": { "Students": { 'schoolCities': 6 } } @@ -193,11 +212,14 @@ class TestLookups(test_engine.EngineTestCase): out_actions = self.update_record("Schools", 4, state="MA") self.assertPartialOutActions(out_actions, { - "calc": [_bulk_update("Students", ["id", "schoolCities" ], [ - [2, "CT,MA" ], - [4, "CT,MA" ], - [6, "CT,MA" ]] - )], + "stored": [ + actions.UpdateRecord("Schools", 4, {"state": "MA"}), + _bulk_update("Students", ["id", "schoolCities" ], [ + [2, "CT,MA" ], + [4, "CT,MA" ], + [6, "CT,MA" ]] + ) + ], "calls": { "Students": { 'schoolCities': 3 } } }) @@ -205,15 +227,24 @@ class TestLookups(test_engine.EngineTestCase): out_actions = self.modify_column("Students", "schoolCities", formula=( "','.join(Schools.lookupRecords(name=$schoolName.upper()).state)")) self.assertPartialOutActions(out_actions, { - "calc": [actions.BulkUpdateRecord("Students", [1,2,3,4,6], - {'schoolCities': ["","","","",""]})], + "stored": [ + actions.ModifyColumn("Students", "schoolCities", { + "formula": "','.join(Schools.lookupRecords(name=$schoolName.upper()).state)" + }), + actions.UpdateRecord("_grist_Tables_column", 6, { + "formula": "','.join(Schools.lookupRecords(name=$schoolName.upper()).state)" + }), + actions.BulkUpdateRecord("Students", [1,2,3,4,6], + {'schoolCities': ["","","","",""]}) + ], "calls": { "Students": { 'schoolCities': 6 } } }) # Changes to dependencies should cause appropriate recalculations. out_actions = self.update_record("Schools", 4, state="KY", name="EUREKA") self.assertPartialOutActions(out_actions, { - "calc": [ + "stored": [ + actions.UpdateRecord("Schools", 4, {"state": "KY", "name": "EUREKA"}), actions.UpdateRecord("Students", 5, {'schoolCities': "KY"}), actions.BulkUpdateRecord("Students", [2,4,6], {'schoolIds': ["3","3","3"]}), ], @@ -239,19 +270,32 @@ class TestLookups(test_engine.EngineTestCase): out_actions = self.add_column("Schools", "lastNames", formula=( "','.join(Students.lookupRecords(schoolName=$name).lastName)")) self.assertPartialOutActions(out_actions, { - "calc": [_bulk_update("Schools", ["id", "lastNames"], [ - [1, "Obama,Clinton"], - [2, "Obama,Clinton"], - [3, "Bush,Bush,Ford"], - [4, "Bush,Bush,Ford"]] - )], + "stored": [ + actions.AddColumn("Schools", "lastNames", { + "formula": "','.join(Students.lookupRecords(schoolName=$name).lastName)", + "isFormula": True, "type": "Any" + }), + actions.AddRecord("_grist_Tables_column", 22, { + "colId": "lastNames", + "formula": "','.join(Students.lookupRecords(schoolName=$name).lastName)", + "isFormula": True, "label": "lastNames", "parentId": 2, "parentPos": 6.0, + "type": "Any", "widgetOptions": "" + }), + _bulk_update("Schools", ["id", "lastNames"], [ + [1, "Obama,Clinton"], + [2, "Obama,Clinton"], + [3, "Bush,Bush,Ford"], + [4, "Bush,Bush,Ford"] + ]), + ], "calls": {"Schools": {"lastNames": 4}, "Students": {"#lookup#schoolName": 6}}, }) # Make sure it responds to changes. out_actions = self.update_record("Students", 5, schoolName="Columbia") self.assertPartialOutActions(out_actions, { - "calc": [ + "stored": [ + actions.UpdateRecord("Students", 5, {"schoolName": "Columbia"}), _bulk_update("Schools", ["id", "lastNames"], [ [1, "Obama,Clinton,Reagan"], [2, "Obama,Clinton,Reagan"]] @@ -269,12 +313,20 @@ class TestLookups(test_engine.EngineTestCase): out_actions = self.modify_column("Schools", "lastNames", formula=( "','.join(Students.lookupRecords(schoolName=$name).firstName)")) self.assertPartialOutActions(out_actions, { - "calc": [_bulk_update("Schools", ["id", "lastNames"], [ - [1, "Barack,Bill,Ronald"], - [2, "Barack,Bill,Ronald"], - [3, "George W,George H,Gerald"], - [4, "George W,George H,Gerald"]] - )], + "stored": [ + actions.ModifyColumn("Schools", "lastNames", { + "formula": "','.join(Students.lookupRecords(schoolName=$name).firstName)" + }), + actions.UpdateRecord("_grist_Tables_column", 22, { + "formula": "','.join(Students.lookupRecords(schoolName=$name).firstName)" + }), + _bulk_update("Schools", ["id", "lastNames"], [ + [1, "Barack,Bill,Ronald"], + [2, "Barack,Bill,Ronald"], + [3, "George W,George H,Gerald"], + [4, "George W,George H,Gerald"]] + ) + ], "calls": {"Schools": {"lastNames": 4}} }) @@ -285,7 +337,8 @@ class TestLookups(test_engine.EngineTestCase): # Make sure that changes still work without errors. out_actions = self.update_record("Students", 5, schoolName="Eureka") self.assertPartialOutActions(out_actions, { - "calc": [ + "stored": [ + actions.UpdateRecord("Students", 5, {"schoolName": "Eureka"}), actions.UpdateRecord("Students", 5, {"schoolCities": ""}), actions.UpdateRecord("Students", 5, {"schoolIds": ""}), ], @@ -326,7 +379,12 @@ return ",".join(str(r.id) for r in Students.lookupRecords(firstName=fn, lastName [4, "Bush,George H"], ]) self.assertPartialOutActions(out_actions, { - "calc": [actions.BulkUpdateRecord("Schools", [2, 4], {"bestStudentId": ["3", "4"]})], + "stored": [ + actions.BulkUpdateRecord("Schools", [2,3,4], { + "bestStudent": ["Clinton,Bill", "Norris,Chuck", "Bush,George H"] + }), + actions.BulkUpdateRecord("Schools", [2, 4], {"bestStudentId": ["3", "4"]}) + ], "calls": {"Schools": {"bestStudentId": 3}} }) self.assertPartialData("Schools", ["id", "bestStudent", "bestStudentId" ], [ @@ -370,7 +428,8 @@ return ",".join(str(r.id) for r in Students.lookupRecords(firstName=fn, lastName out_actions = self.remove_record("Schools", 3) self.assertPartialOutActions(out_actions, { - "calc": [ + "stored": [ + actions.RemoveRecord("Schools", 3), actions.BulkUpdateRecord("Students", [2,4,6], { "schoolCities": ["West Haven","West Haven","West Haven"]}), actions.BulkUpdateRecord("Students", [2,4,6], { @@ -452,7 +511,8 @@ return ",".join(str(r.id) for r in Students.lookupRecords(firstName=fn, lastName # test, even though schoolIds depends on name indirectly. out_actions = self.update_record("Schools", 2, name="Eureka") self.assertPartialOutActions(out_actions, { - "calc": [ + "stored": [ + actions.UpdateRecord("Schools", 2, {"name": "Eureka"}), actions.UpdateRecord("Schools", 2, {"cname": "Eureka"}), actions.BulkUpdateRecord("Students", [1,3,5], { 'schoolCities': ["New York", "New York", "Colombia"] @@ -503,14 +563,15 @@ return ",".join(str(r.id) for r in Students.lookupRecords(firstName=fn, lastName self.use_saved_lookup_results() out_actions = self.update_record("Schools", 2, name="Eureka") self.assertPartialOutActions(out_actions, { - "calc": [ - actions.BulkUpdateRecord('Students', [1,3,5], {'schools': [[1],[1],[2]]}), + "stored": [ + actions.UpdateRecord('Schools', 2, {'name': "Eureka"}), actions.BulkUpdateRecord("Students", [1,3,5], { 'schoolCities': ["New York", "New York", "Colombia"] }), actions.BulkUpdateRecord("Students", [1,3,5], { 'schoolIds': ["1", "1","2"] }), + actions.BulkUpdateRecord('Students', [1,3,5], {'schools': [[1],[1],[2]]}), ], "calls": {"Students": { 'schools': 3, 'schoolCities': 3, 'schoolIds': 3 }, "Schools": {'#lookup#name': 1} }, @@ -522,10 +583,11 @@ return ",".join(str(r.id) for r in Students.lookupRecords(firstName=fn, lastName [5, "Yale"] ]) self.assertPartialOutActions(out_actions, { - "calc": [ - actions.BulkUpdateRecord("Students", [3,5], {'schools': [[], [3,4]]}), + "stored": [ + actions.BulkUpdateRecord("Students", [3,5], {'schoolName': ["", "Yale"]}), actions.BulkUpdateRecord("Students", [3,5], {'schoolCities': ["", "New Haven:West Haven"]}), actions.BulkUpdateRecord("Students", [3,5], {'schoolIds': ["", "3:4"]}), + actions.BulkUpdateRecord("Students", [3,5], {'schools': [[], [3,4]]}), ], "calls": { "Students": { 'schools': 2, 'schoolCities': 2, 'schoolIds': 2 } }, }) diff --git a/sandbox/grist/test_renames.py b/sandbox/grist/test_renames.py index 8cd488c6..e4126bf8 100644 --- a/sandbox/grist/test_renames.py +++ b/sandbox/grist/test_renames.py @@ -225,7 +225,14 @@ class TestRenames(test_engine.EngineTestCase): "formula": ["", "People.lookupOne(addr=$id, city=$city).nombre", "People.lookupRecords(addr=$id).nombre"] - }] + }], + # TODO This is a symptom of comparing before and after values using rich values that refer + # to a destroyed column (a ColumnView). In reality, the values before and after after the + # same, but here the attempt to encode the previous value produces an incorrect result. + # (It's a bug, but not easy to fix and hopefully hard to run into.) + ["BulkUpdateRecord", "Address", [11, 12, 13], + {"people2": [["L", "Sam"], ["L", "Bob", "Doug"], ["L", "Alice"]] + }], ]}) def test_rename_lookup_ref_attr(self): diff --git a/sandbox/grist/test_renames2.py b/sandbox/grist/test_renames2.py index 50574199..71790f5c 100644 --- a/sandbox/grist/test_renames2.py +++ b/sandbox/grist/test_renames2.py @@ -1,5 +1,5 @@ -import logger import textwrap +import logger import test_engine @@ -202,6 +202,10 @@ class TestRenames2(test_engine.EngineTestCase): "formula": [ "", "' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))"] }], + ["BulkUpdateRecord", "Games", [1, 2, 3, 4], { + "win4_game_name": [["E", "AttributeError"], ["E", "AttributeError"], + ["E", "AttributeError"], ["E", "AttributeError"]] + }], ]}) # Fix up things missed due to the TODOs above. @@ -272,7 +276,15 @@ class TestRenames2(test_engine.EngineTestCase): ["BulkUpdateRecord", "_grist_Tables_column", [2, 11], { "colId": ["nombre", "N"], "formula": ["", "$nombre.upper()"] - }] + }], + ["BulkUpdateRecord", "Games", [1, 2, 3, 4], { + "win3_person_name": [["E", "AttributeError"], ["E", "AttributeError"], + ["E", "AttributeError"], ["E", "AttributeError"]] + }], + ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5], { + "PartnerNames": [["E", "AttributeError"], ["E", "AttributeError"], + ["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"]] + }], ]}) # Fix up things missed due to the TODOs above. @@ -298,7 +310,11 @@ class TestRenames2(test_engine.EngineTestCase): ["BulkUpdateRecord", "_grist_Tables_column", [14, 15], { "colId": ["companero", "partner4"], "formula": [self.partner, "$companero.companero.partner.partner"] - }] + }], + ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5], { + "partner4": [["E", "AttributeError"], ["E", "AttributeError"], + ["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"]] + }], ]}) # Fix up things missed due to the TODOs above. @@ -321,7 +337,10 @@ class TestRenames2(test_engine.EngineTestCase): "colId": ["pwin", "win3_person_name", "win4_game_name"], "formula": ["Entries.lookupOne(person=$id, rank=1).game", "$win.pwin.win.name", "$win.pwin.win.win.name"]}], - + ["BulkUpdateRecord", "Games", [1, 2, 3, 4], { + "win4_game_name": [["E", "AttributeError"], ["E", "AttributeError"], + ["E", "AttributeError"], ["E", "AttributeError"]] + }], ]}) # Fix up things missed due to the TODOs above. diff --git a/sandbox/grist/test_side_effects.py b/sandbox/grist/test_side_effects.py index b18a5902..55609651 100644 --- a/sandbox/grist/test_side_effects.py +++ b/sandbox/grist/test_side_effects.py @@ -48,6 +48,7 @@ class TestSideEffects(test_engine.EngineTestCase): "colId": "A", "formula": formula, "isFormula": True, "label": "A", "parentId": 1, "parentPos": 4.0, "type": "Any", "widgetOptions": "" }], + ["BulkUpdateRecord", "Address", [21, 22], {"A": [["E", "Exception"], ["E", "Exception"]]}], # The thing to note here is that while lookupOrAddDerived() should have added a row to # Schools, the Exception negated it, and there is no action to add that row. ]}) @@ -86,8 +87,6 @@ if $amount < 0: "stored": [ ["UpdateRecord", "Address", 22, {"amount": 1000.0, "city": "aaa"}], ["AddRecord", "Schools", 4, {"city": "aaa"}], - ], - "calc": [ ["UpdateRecord", "Schools", 4, {"ucity": "AAA"}], ], }) @@ -105,8 +104,6 @@ if $amount < 0: self.assertPartialOutActions(out_actions, { "stored": [ ["UpdateRecord", "Address", 22, {"amount": -3.0, "city": "bbb"}], - ], - "calc": [ ["UpdateRecord", "Address", 22, {"A": ["E", "Exception"]}], ], }) diff --git a/sandbox/grist/test_summary.py b/sandbox/grist/test_summary.py index bdefb110..9448b63a 100644 --- a/sandbox/grist/test_summary.py +++ b/sandbox/grist/test_summary.py @@ -446,42 +446,44 @@ class Address: # Change an amount (New York, NY, 6 -> 106), check that the right calc action gets emitted. out_actions = self.update_record(source_tbl_name, 26, amount=106) self.assertPartialOutActions(out_actions, { - "stored": [actions.UpdateRecord(source_tbl_name, 26, {'amount': 106})], - "calc": [actions.UpdateRecord(summary_tbl_name, 1, {'amount': 1.+106+11})] + "stored": [ + actions.UpdateRecord(source_tbl_name, 26, {'amount': 106}), + actions.UpdateRecord(summary_tbl_name, 1, {'amount': 1.+106+11}), + ] }) # Change a groupby value so that a record moves from one summary group to another. # Bedford, NY, 8.0 -> Bedford, MA, 8.0 out_actions = self.update_record(source_tbl_name, 28, state="MA") self.assertPartialOutActions(out_actions, { - "stored": [actions.UpdateRecord(source_tbl_name, 28, {'state': 'MA'})], - "calc": [ - actions.BulkUpdateRecord(summary_tbl_name, [5,7], {'group': [[25, 28], []]}), + "stored": [ + actions.UpdateRecord(source_tbl_name, 28, {'state': 'MA'}), actions.BulkUpdateRecord(summary_tbl_name, [5,7], {'amount': [5.0 + 8.0, 0.0]}), actions.BulkUpdateRecord(summary_tbl_name, [5,7], {'count': [2, 0]}), + actions.BulkUpdateRecord(summary_tbl_name, [5,7], {'group': [[25, 28], []]}), ] }) # Add a record to an existing group (Bedford, MA, 108.0) out_actions = self.add_record(source_tbl_name, city="Bedford", state="MA", amount=108.0) self.assertPartialOutActions(out_actions, { - "stored": [actions.AddRecord(source_tbl_name, 32, - {'city': 'Bedford', 'state': 'MA', 'amount': 108.0})], - "calc": [ - actions.UpdateRecord(summary_tbl_name, 5, {'group': [25, 28, 32]}), + "stored": [ + actions.AddRecord(source_tbl_name, 32, + {'city': 'Bedford', 'state': 'MA', 'amount': 108.0}), actions.UpdateRecord(summary_tbl_name, 5, {'amount': 5.0 + 8.0 + 108.0}), actions.UpdateRecord(summary_tbl_name, 5, {'count': 3}), + actions.UpdateRecord(summary_tbl_name, 5, {'group': [25, 28, 32]}), ] }) # Remove a record (rowId=28, Bedford, MA, 8.0) out_actions = self.remove_record(source_tbl_name, 28) self.assertPartialOutActions(out_actions, { - "stored": [actions.RemoveRecord(source_tbl_name, 28)], - "calc": [ - actions.UpdateRecord(summary_tbl_name, 5, {'group': [25, 32]}), + "stored": [ + actions.RemoveRecord(source_tbl_name, 28), actions.UpdateRecord(summary_tbl_name, 5, {'amount': 5.0 + 108.0}), actions.UpdateRecord(summary_tbl_name, 5, {'count': 2}), + actions.UpdateRecord(summary_tbl_name, 5, {'group': [25, 32]}), ] }) @@ -492,11 +494,9 @@ class Address: "stored": [ actions.UpdateRecord(source_tbl_name, 25, {'city': 'Salem'}), actions.AddRecord(summary_tbl_name, 10, {'city': 'Salem', 'state': 'MA'}), - ], - "calc": [ - actions.BulkUpdateRecord(summary_tbl_name, [5,10], {'group': [[32], [25]]}), actions.BulkUpdateRecord(summary_tbl_name, [5,10], {'amount': [108.0, 5.0]}), actions.BulkUpdateRecord(summary_tbl_name, [5,10], {'count': [1, 1]}), + actions.BulkUpdateRecord(summary_tbl_name, [5,10], {'group': [[32], [25]]}), ] }) @@ -506,11 +506,9 @@ class Address: "stored": [ actions.AddRecord(source_tbl_name, 33, {'city': 'Amherst', 'state': 'MA', 'amount': 17.}), actions.AddRecord(summary_tbl_name, 11, {'city': 'Amherst', 'state': 'MA'}), - ], - "calc": [ - actions.UpdateRecord(summary_tbl_name, 11, {'group': [33]}), actions.UpdateRecord(summary_tbl_name, 11, {'amount': 17.0}), actions.UpdateRecord(summary_tbl_name, 11, {'count': 1}), + actions.UpdateRecord(summary_tbl_name, 11, {'group': [33]}), ] }) diff --git a/sandbox/grist/test_summary2.py b/sandbox/grist/test_summary2.py index bdb3422f..15442241 100644 --- a/sandbox/grist/test_summary2.py +++ b/sandbox/grist/test_summary2.py @@ -141,19 +141,19 @@ class TestSummary2(test_engine.EngineTestCase): # Modify a value, and check that various tables got updated correctly. out_actions = self.update_record("Address", 28, state="MA") self.assertPartialOutActions(out_actions, { - "stored": [actions.UpdateRecord("Address", 28, {'state': 'MA'})], - "calc": [ - actions.BulkUpdateRecord("GristSummary_7_Address", [5,7], {'group': [[25, 28], []]}), + "stored": [ + actions.UpdateRecord("Address", 28, {'state': 'MA'}), actions.BulkUpdateRecord("GristSummary_7_Address", [5,7], {'amount': [5.0 + 8.0, 0.0]}), - actions.BulkUpdateRecord("GristSummary_7_Address", [5,7], {'count': [2, 0]}), actions.BulkUpdateRecord("GristSummary_7_Address", [5,7], {'average': [6.5, objtypes.RaisedException(ZeroDivisionError())]}), + actions.BulkUpdateRecord("GristSummary_7_Address", [5,7], {'count': [2, 0]}), + actions.BulkUpdateRecord("GristSummary_7_Address", [5,7], {'group': [[25, 28], []]}), actions.UpdateRecord("GristSummary_7_Address3", 5, {'state': "MA"}), - actions.BulkUpdateRecord("GristSummary_7_Address4", [1,4], - {'group': [[21,22,26,27,30,31], [25,28,29]]}), actions.BulkUpdateRecord("GristSummary_7_Address4", [1,4], {'amount': [1.+2+6+7+10+11, 5.+8+9]}), actions.BulkUpdateRecord("GristSummary_7_Address4", [1,4], {'count': [6, 3]}), + actions.BulkUpdateRecord("GristSummary_7_Address4", [1,4], + {'group': [[21,22,26,27,30,31], [25,28,29]]}), ] }) diff --git a/sandbox/grist/test_types.py b/sandbox/grist/test_types.py index 8396ae5e..afeb492c 100644 --- a/sandbox/grist/test_types.py +++ b/sandbox/grist/test_types.py @@ -66,14 +66,18 @@ class TestTypes(test_engine.EngineTestCase): "int": [None, None, 1, 0, 8, 1509556595, 1, 0, "Chîcágö", "New York"], "bool": [False, False, True, False, True, True, True, False, "Chîcágö", "New York"], "date": [None, None, 1.0, 0.0, 8.153, 1509556595.0, 1.0, 0.0, 1548115200.0, "New York"] - }]], + }], + ["UpdateRecord", "Formulas", 1, {"division": 0.0}], + ], "undo": [["BulkUpdateRecord", "Types", self.all_row_ids, { "text": ["New York", "Chîcágö", False, True, 1509556595, 8.153, 0, 1, "", None], "numeric": ["New York", "Chîcágö", False, True, 1509556595, 8.153, 0, 1, "", None], "int": ["New York", "Chîcágö", False, True, 1509556595, 8.153, 0, 1, "", None], "bool": ["New York", "Chîcágö", False, True, 1509556595, 8.153, False, True, "", None], "date": ["New York", "Chîcágö", False, True, 1509556595, 8.153, 0, 1, "", None] - }]] + }], + ["UpdateRecord", "Formulas", 1, {"division": 0.5}], + ] }) self.assertTableData("Types", data=[ @@ -114,12 +118,14 @@ class TestTypes(test_engine.EngineTestCase): ["BulkUpdateRecord", "Types", [13, 14, 15, 16, 17, 18], {"numeric": ["False", "True", "1509556595.0", "8.153", "0.0", "1.0"]}], ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Text"}], + ["UpdateRecord", "Formulas", 1, {"division": ["E", "TypeError"]}], ], "undo": [ ["BulkUpdateRecord", "Types", [13, 14, 15, 16, 17, 18], {"numeric": [False, True, 1509556595, 8.153, 0, 1]}], ["ModifyColumn", "Types", "numeric", {"type": "Numeric"}], ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Numeric"}], + ["UpdateRecord", "Formulas", 1, {"division": 0.5}], ] }) @@ -321,12 +327,14 @@ class TestTypes(test_engine.EngineTestCase): ["BulkUpdateRecord", "Types", [13, 14, 15, 16, 17, 18, 19], {"numeric": [0, 1, 1509556595, 8, 0, 1, None]}], ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Int"}], + ["UpdateRecord", "Formulas", 1, {"division": 0}], ], "undo": [ ["BulkUpdateRecord", "Types", [13, 14, 15, 16, 17, 18, 19], {"numeric": [False, True, 1509556595.0, 8.153, 0.0, 1.0, ""]}], ["ModifyColumn", "Types", "numeric", {"type": "Numeric"}], ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Numeric"}], + ["UpdateRecord", "Formulas", 1, {"division": 0.5}], ] }) @@ -419,12 +427,14 @@ class TestTypes(test_engine.EngineTestCase): ["BulkUpdateRecord", "Types", [15, 16, 17, 18, 19, 20], {"numeric": [True, True, False, True, False, False]}], ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Bool"}], + ["UpdateRecord", "Formulas", 1, {"division": 0}], ], "undo": [ ["BulkUpdateRecord", "Types", [15, 16, 17, 18, 19, 20], {"numeric": [1509556595.0, 8.153, 0.0, 1.0, "", None]}], ["ModifyColumn", "Types", "numeric", {"type": "Numeric"}], ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Numeric"}], + ["UpdateRecord", "Formulas", 1, {"division": 0.5}], ] }) @@ -518,12 +528,14 @@ class TestTypes(test_engine.EngineTestCase): ["BulkUpdateRecord", "Types", [13, 14, 19], {"numeric": [0.0, 1.0, None]}], ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Date"}], + ["UpdateRecord", "Formulas", 1, {"division": ["E", "TypeError"]}], ], "undo": [ ["BulkUpdateRecord", "Types", [13, 14, 19], {"numeric": [False, True, ""]}], ["ModifyColumn", "Types", "numeric", {"type": "Numeric"}], ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Numeric"}], + ["UpdateRecord", "Formulas", 1, {"division": 0.5}], ] }) diff --git a/sandbox/grist/test_useractions.py b/sandbox/grist/test_useractions.py index 1dc8f8b8..757a4af8 100644 --- a/sandbox/grist/test_useractions.py +++ b/sandbox/grist/test_useractions.py @@ -77,6 +77,8 @@ class TestUserActions(test_engine.EngineTestCase): ["AddRecord", "_grist_Views_section_field", 2, { "colRef": 24, "parentId": 1, "parentPos": 2.0 }], + ["BulkUpdateRecord", "Schools", [1, 2, 3], + {"grist_Transform": ["New York", "Colombia", "New York"]}], ]}) out_actions = self.update_record('_grist_Tables_column', 24, @@ -87,6 +89,7 @@ class TestUserActions(test_engine.EngineTestCase): 'formula': 'return Address.lookupOne(city=$city).id', 'type': 'Ref:Address'}], ['UpdateRecord', '_grist_Tables_column', 24, { 'formula': 'return Address.lookupOne(city=$city).id', 'type': 'Ref:Address'}], + ["BulkUpdateRecord", "Schools", [1, 2, 3, 4], {"grist_Transform": [11, 12, 11, 0]}], ]}) # It seems best if TypeTransform sets widgetOptions on grist_Transform column, so that they @@ -104,6 +107,7 @@ class TestUserActions(test_engine.EngineTestCase): 'type': 'Ref:Address', 'widgetOptions': 'world' }], ['BulkUpdateRecord', 'Schools', [1, 2, 3], {'city': [11, 12, 11]}], + ["BulkUpdateRecord", "Schools", [1, 2, 3], {"grist_Transform": [0, 0, 0]}], ]}) out_actions = self.update_record('_grist_Tables_column', 23, diff --git a/sandbox/grist/testscript.json b/sandbox/grist/testscript.json index e2a9cd2f..1d31bd4a 100644 --- a/sandbox/grist/testscript.json +++ b/sandbox/grist/testscript.json @@ -97,9 +97,10 @@ "city": "Washington", "state": "DC", "country": "US" - }]], + }], + ["UpdateRecord", "Address", 11, {"region": "North America"}] + ], "undo": [["RemoveRecord", "Address", 11]], - "calc": [["UpdateRecord", "Address", 11, {"region": "North America"}]], "retValue": [ 11 ] }, "CHECK_CALL_COUNTS": { @@ -125,11 +126,15 @@ // Set a student to the new school, and ensure formulas see the correct school and address. "USER_ACTION": ["UpdateRecord", "Students", 3, {"school": 9}], "ACTIONS": { - "stored": [["UpdateRecord", "Students", 3, {"school": 9}]], - "undo": [["UpdateRecord", "Students", 3, {"school": 6}]], - "calc": [ + "stored": [ + ["UpdateRecord", "Students", 3, {"school": 9}], ["UpdateRecord", "Students", 3, {"schoolRegion": "DC"}], ["UpdateRecord", "Students", 3, {"schoolShort": "Georgetown"}] + ], + "undo": [ + ["UpdateRecord", "Students", 3, {"school": 6}], + ["UpdateRecord", "Students", 3, {"schoolRegion": "Europe"}], + ["UpdateRecord", "Students", 3, {"schoolShort": "Oxford"}] ] }, "CHECK_CALL_COUNTS": { @@ -269,8 +274,7 @@ ["RemoveRecord", "Schools", 13], ["RemoveRecord", "Schools", 14], ["RemoveRecord", "Schools", 15], - ["BulkUpdateRecord", "Schools", [1, 2, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15], - {"numStudents": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "1000000000000000000000000000", 1000000]}], + // No updates in undo for Schools.numStudents column, because the undo removes this column. ["ModifyColumn", "Schools", "numStudents", {"type": "Int"}], ["UpdateRecord", "_grist_Tables_column", 30, {"type": "Int"}], ["RemoveRecord", "Schools", 16] @@ -334,12 +338,12 @@ "city": ["Washington", "Portland"], "state": ["DC", "OR"] }], - ["BulkUpdateRecord", "Address", [11, 12], {"country": ["US", "US"]}] + ["BulkUpdateRecord", "Address", [11, 12], {"country": ["US", "US"]}], + ["BulkUpdateRecord", "Address", [11, 12], { + "region": ["North America", "North America"] + }] ], "undo": [["BulkRemoveRecord", "Address", [11, 12]]], - "calc": [["BulkUpdateRecord", "Address", [11, 12], { - "region": ["North America", "North America"] - }]], "retValue": [ [11, 12] ] }, "CHECK_CALL_COUNTS" : { @@ -369,11 +373,15 @@ ["APPLY", { "USER_ACTION": ["UpdateRecord", "Address", 10, {"country": "GB"}], "ACTIONS": { - "stored": [["UpdateRecord", "Address", 10, {"country": "GB"}]], - "undo": [["UpdateRecord", "Address", 10, {"country": "UK"}]], - "calc": [ + "stored": [ + ["UpdateRecord", "Address", 10, {"country": "GB"}], ["UpdateRecord", "Address", 10, {"region": "N/A"}], ["UpdateRecord", "Students", 3, {"schoolRegion": "N/A"}] + ], + "undo": [ + ["UpdateRecord", "Address", 10, {"country": "UK"}], + ["UpdateRecord", "Address", 10, {"region": "Europe"}], + ["UpdateRecord", "Students", 3, {"schoolRegion": "Europe"}] ] }, "CHECK_CALL_COUNTS" : { @@ -387,11 +395,15 @@ {"name": ["eureka", "columbia", "yale"]}], "ACTIONS": { "stored": [["BulkUpdateRecord", "Schools", [1, 2, 8], - {"name": ["eureka", "columbia", "yale"]}]], + {"name": ["eureka", "columbia", "yale"]}], + ["BulkUpdateRecord", "Students", [1, 2, 4, 5], + {"schoolShort": ["columbia", "yale", "yale", "eureka"]}] + ], "undo": [["BulkUpdateRecord", "Schools", [1, 2, 8], - {"name": ["Eureka College", "Columbia University", "Yale University"]}]], - "calc": [["BulkUpdateRecord", "Students", [1, 2, 4, 5], - {"schoolShort": ["columbia", "yale", "yale", "eureka"]}]] + {"name": ["Eureka College", "Columbia University", "Yale University"]}], + ["BulkUpdateRecord", "Students", [1, 2, 4, 5], + {"schoolShort": ["Columbia", "Yale", "Yale", "Eureka"]}] + ] }, "CHECK_CALL_COUNTS" : { "Students" : { "schoolShort" : 4 } @@ -441,7 +453,9 @@ ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "str.upper(rec.firstName) + ' ' + rec.lastName"}], ["RenameColumn", "Students", "schoolShort", "shortSchool"], - ["UpdateRecord", "_grist_Tables_column", 6, {"colId": "shortSchool"}] + ["UpdateRecord", "_grist_Tables_column", 6, {"colId": "shortSchool"}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"fullName": ["BARACK Obama", "GEORGE W Bush", "BILL Clinton", "GEORGE H Bush", "RONALD Reagan", "JIMMY Carter", "GERALD Ford"]}] ], "undo": [ ["ModifyColumn", "Students", "fullName", @@ -449,11 +463,9 @@ ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "rec.firstName + ' ' + rec.lastName"}], ["RenameColumn", "Students", "shortSchool", "schoolShort"], - ["UpdateRecord", "_grist_Tables_column", 6, {"colId": "schoolShort"}] - ], - "calc": [ + ["UpdateRecord", "_grist_Tables_column", 6, {"colId": "schoolShort"}], ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], - {"fullName": ["BARACK Obama", "GEORGE W Bush", "BILL Clinton", "GEORGE H Bush", "RONALD Reagan", "JIMMY Carter", "GERALD Ford"]}] + {"fullName": ["Barack Obama", "George W Bush", "Bill Clinton", "George H Bush", "Ronald Reagan", "Jimmy Carter", "Gerald Ford"]}] ] }, "CHECK_CALL_COUNTS" : { @@ -469,13 +481,12 @@ ], "ACTIONS" : { "stored" : [ - ["UpdateRecord", "Students", 2, {"firstName" : "Richard", "lastName" : "Nixon"}] + ["UpdateRecord", "Students", 2, {"firstName" : "Richard", "lastName" : "Nixon"}], + ["UpdateRecord", "Students", 2, {"fullName" : "RICHARD Nixon"}] ], "undo" : [ - ["UpdateRecord", "Students", 2, {"firstName" : "George W", "lastName" : "Bush"}] - ], - "calc" : [ - ["UpdateRecord", "Students", 2, {"fullName" : "RICHARD Nixon"}] + ["UpdateRecord", "Students", 2, {"firstName" : "George W", "lastName" : "Bush"}], + ["UpdateRecord", "Students", 2, {"fullName": "GEORGE W Bush"}] ] }, "CHECK_CALL_COUNTS" : { @@ -526,13 +537,14 @@ "ACTIONS": { "stored": [ ["RemoveRecord", "Address", 4], - ["BulkUpdateRecord", "Schools", [7, 8], {"address": [0, 0]}] + ["BulkUpdateRecord", "Schools", [7, 8], {"address": [0, 0]}], + ["BulkUpdateRecord", "Students", [2, 4, 8], {"schoolRegion": [null, null, null]}] ], "undo": [ - ["AddRecord", "Address", 4, {"city": "New Haven", "state": "CT", "country": "US"}], - ["BulkUpdateRecord", "Schools", [7, 8], {"address": [4, 4]}] - ], - "calc": [["BulkUpdateRecord", "Students", [2, 4, 8], {"schoolRegion": [null, null, null]}]] + ["AddRecord", "Address", 4, {"city": "New Haven", "country": "US", "region": "North America", "state": "CT"}], + ["BulkUpdateRecord", "Schools", [7, 8], {"address": [4, 4]}], + ["BulkUpdateRecord", "Students", [2, 4, 8], {"schoolRegion": ["CT", "CT", "CT"]}] + ] }, "CHECK_CALL_COUNTS" : { "Students" : { "schoolRegion" : 3 } @@ -544,15 +556,15 @@ "ACTIONS": { "stored": [ ["RemoveRecord", "Schools", 5], - ["UpdateRecord", "Students", 6, {"school": 0}] + ["UpdateRecord", "Students", 6, {"school": 0}], + ["UpdateRecord", "Students", 6, {"schoolRegion": null}], + ["UpdateRecord", "Students", 6, {"schoolShort": ""}] ], "undo": [ ["AddRecord", "Schools", 5, {"name": "U.S. Naval Academy", "address": 3}], - ["UpdateRecord", "Students", 6, {"school": 5}] - ], - "calc": [ - ["UpdateRecord", "Students", 6, {"schoolRegion": null}], - ["UpdateRecord", "Students", 6, {"schoolShort": ""}] + ["UpdateRecord", "Students", 6, {"school": 5}], + ["UpdateRecord", "Students", 6, {"schoolRegion": "MD"}], + ["UpdateRecord", "Students", 6, {"schoolShort": "U.S."}] ] }, "CHECK_CALL_COUNTS" : { @@ -565,7 +577,8 @@ "ACTIONS": { "stored": [["RemoveRecord", "Students", 1]], "undo": [["AddRecord", "Students", 1, - {"firstName": "Barack", "lastName": "Obama", "school": 2}]] + {"firstName": "Barack", "fullName": "Barack Obama", "fullNameLen": 12, "lastName": "Obama", "school": 2, "schoolRegion": "NY", "schoolShort": "Columbia"}] + ] } }], @@ -609,7 +622,11 @@ "undo": [["BulkAddRecord", "Students", [2, 5, 6, 8], { "firstName": ["George W", "Ronald", "Jimmy", "Gerald"], "lastName": ["Bush", "Reagan", "Carter", "Ford"], - "school": [8, 1, 5, 7] + "school": [8, 1, 5, 7], + "fullName": ["George W Bush", "Ronald Reagan", "Jimmy Carter", "Gerald Ford"], + "fullNameLen": [13, 13, 12, 11], + "schoolRegion": ["CT", "IL", "MD", "CT"], + "schoolShort": ["Yale", "Eureka", "U.S.", "Yale"] }]] } }], @@ -620,18 +637,18 @@ "ACTIONS": { "stored": [ ["BulkRemoveRecord", "Schools", [6, 8]], - ["BulkUpdateRecord", "Students", [3, 4], {"school": [0, 0]}] + ["BulkUpdateRecord", "Students", [3, 4], {"school": [0, 0]}], + ["BulkUpdateRecord", "Students", [3, 4], {"schoolRegion": [null, null]}], + ["BulkUpdateRecord", "Students", [3, 4], {"schoolShort": ["", ""]}] ], "undo": [ ["BulkAddRecord", "Schools", [6, 8], { "name": ["Oxford University", "Yale University"], "address": [10, 4] }], - ["BulkUpdateRecord", "Students", [3, 4], {"school": [6, 8]}] - ], - "calc": [ - ["BulkUpdateRecord", "Students", [3, 4], {"schoolRegion": [null, null]}], - ["BulkUpdateRecord", "Students", [3, 4], {"schoolShort": ["", ""]}] + ["BulkUpdateRecord", "Students", [3, 4], {"school": [6, 8]}], + ["BulkUpdateRecord", "Students", [3, 4], {"schoolRegion": ["Europe", "CT"]}], + ["BulkUpdateRecord", "Students", [3, 4], {"schoolShort": ["Oxford", "Yale"]}] ] }, "CHECK_CALL_COUNTS" : { @@ -753,17 +770,15 @@ "formula": "rec.zip[:5]", "label": "zip5", "widgetOptions": "" - }] + }], + // Since it's a formula, it immediately gets evaluated, causing some calc actions. + ["BulkUpdateRecord", "Address", [2, 3, 4, 7, 10], + {"zip5": ["61530", "", "06520", "10027", ""]}] ], "undo": [ ["RemoveColumn", "Address", "zip5"], ["RemoveRecord", "_grist_Tables_column", 31] ], - // Since it's a formula, it immediately gets evaluated, causing some calc actions. - "calc": [ - ["BulkUpdateRecord", "Address", [2, 3, 4, 7, 10], - {"zip5": ["61530", "", "06520", "10027", ""]}] - ], "retValue": [ {"colId": "zip5", "colRef": 31} ] @@ -800,16 +815,14 @@ "formula": "rec.address.zip[:5]", "label": "zippy", "widgetOptions": "" - }] + }], + ["BulkUpdateRecord", "Schools", [1,2,5,6,7,8], + {"zip": ["61530", "10027", "", "", "06520", "06520"]}] ], "undo": [ ["RemoveColumn", "Schools", "zip"], ["RemoveRecord", "_grist_Tables_column", 32] ], - "calc": [ - ["BulkUpdateRecord", "Schools", [1,2,5,6,7,8], - {"zip": ["61530", "10027", "", "", "06520", "06520"]}] - ], "retValue": [ {"colId": "zip", "colRef": 32} ] @@ -889,7 +902,6 @@ "isFormula": true, "label": "world", "widgetOptions": ""}], ["AddRecord", "_grist_Views_section_field", 2, {"colRef": 35, "parentId": 1, "parentPos": 2.0}] ], - "calc": [], "undo": [ ["RemoveTable", "Bar"], ["RemoveRecord", "_grist_Tables", 4], @@ -952,8 +964,7 @@ "USER_ACTION": ["UpdateRecord", "Address", 3, {"city": ""}], "ACTIONS": { "stored": [["UpdateRecord", "Address", 3, {"city": ""}]], - "undo": [["UpdateRecord", "Address", 3, {"city": "Annapolis"}]], - "calc": [] + "undo": [["UpdateRecord", "Address", 3, {"city": "Annapolis"}]] } }], @@ -1003,6 +1014,7 @@ ["RemoveColumn", "Students", "fullNameLen"] ], "undo": [ + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], {"fullNameLen": [12, 13, 12, 13, 13, 12, 11]}], ["AddRecord", "_grist_Tables_column", 4, { "parentId": 1, "parentPos": 4.0, @@ -1048,7 +1060,12 @@ "ACTIONS": { "stored": [ ["RemoveRecord", "_grist_Tables_column", 27], - ["RemoveColumn", "Address", "state"] + ["RemoveColumn", "Address", "state"], + ["BulkUpdateRecord", "Students", [1, 2, 4, 5, 6, 8], + {"schoolRegion": [["E","AttributeError"], ["E","AttributeError"], + ["E","AttributeError"], ["E","AttributeError"], ["E","AttributeError"], + ["E","AttributeError"]] + }] ], "undo": [ ["AddRecord", "_grist_Tables_column", 27, { @@ -1064,15 +1081,8 @@ "type": "Text", "isFormula": false, "formula": "" - }] - - ], - "calc": [ - ["BulkUpdateRecord", "Students", [1, 2, 4, 5, 6, 8], - {"schoolRegion": [["E","AttributeError"], ["E","AttributeError"], - ["E","AttributeError"], ["E","AttributeError"], ["E","AttributeError"], - ["E","AttributeError"]] - }] + }], + ["BulkUpdateRecord", "Students", [1, 2, 4, 5, 6, 8], {"schoolRegion": ["NY", "CT", "CT", "IL", "MD", "CT"]}] ] }, "CHECK_CALL_COUNTS": { @@ -1116,9 +1126,12 @@ ["RemoveRecord", "_grist_Tables_column", 6], ["RemoveColumn", "Students", "schoolShort"], ["RemoveRecord", "Schools", 1], - ["RemoveRecord", "Schools", 8] + ["RemoveRecord", "Schools", 8], + ["UpdateRecord", "Students", 3, + {"schoolRegion": ["E","AttributeError"]}] ], "undo": [ + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], {"schoolShort": ["Columbia", "Yale", "Oxford", "Yale", "Eureka", "U.S.", "Yale"]}], ["AddRecord", "_grist_Tables_column", 5, {"parentPos": 5.0, "parentId": 1, "colId": "school", "type": "Ref:Schools", "label": "school" }], @@ -1133,11 +1146,8 @@ ["AddColumn", "Students", "schoolShort", {"isFormula": true, "formula": "rec.school.name.split(' ')[0]", "type": "Any"}], ["AddRecord", "Schools", 1, {"name": "Eureka College", "address": 2}], - ["AddRecord", "Schools", 8, {"name": "Yale University", "address": 4}] - ], - "calc": [ - ["UpdateRecord", "Students", 3, - {"schoolRegion": ["E","AttributeError"]}] + ["AddRecord", "Schools", 8, {"name": "Yale University", "address": 4}], + ["UpdateRecord", "Students", 3, {"schoolRegion": "Europe"}] ] }, "CHECK_CALL_COUNTS": { @@ -1214,7 +1224,6 @@ ["RemoveColumn", "ViewTest", "hello"] ], - "calc": [], "undo": [ ["RemoveTable", "ViewTest"], ["RemoveRecord", "_grist_Tables", 4], @@ -1299,10 +1308,6 @@ ["RemoveRecord", "Schools", 8] ], "ACTIONS": { - "calc": [ - ["BulkUpdateRecord", "Students", [2, 4, 5], {"schoolRegion": [null, null, null]}], - ["BulkUpdateRecord", "Students", [2, 4, 5], {"schoolShort": ["", "", ""]}] - ], "stored": [ ["RenameColumn", "Students", "school", "university"], ["ModifyColumn", "Students", "schoolShort", @@ -1321,7 +1326,9 @@ ["RemoveRecord", "Schools", 1], ["UpdateRecord", "Students", 5, {"university": 0}], ["RemoveRecord", "Schools", 8], - ["BulkUpdateRecord", "Students", [2, 4], {"university": [0, 0]}] + ["BulkUpdateRecord", "Students", [2, 4], {"university": [0, 0]}], + ["BulkUpdateRecord", "Students", [2, 4, 5], {"schoolRegion": [null, null, null]}], + ["BulkUpdateRecord", "Students", [2, 4, 5], {"schoolShort": ["", "", ""]}] ], "undo": [ ["RenameColumn", "Students", "university", "school"], @@ -1341,7 +1348,9 @@ ["AddRecord", "Schools", 1, {"name": "Eureka College", "address": 2}], ["UpdateRecord", "Students", 5, {"university": 1}], ["AddRecord", "Schools", 8, {"name": "Yale University", "address": 4}], - ["BulkUpdateRecord", "Students", [2, 4], {"university": [8, 8]}] + ["BulkUpdateRecord", "Students", [2, 4], {"university": [8, 8]}], + ["BulkUpdateRecord", "Students", [2, 4, 5], {"schoolRegion": ["CT", "CT", "IL"]}], + ["BulkUpdateRecord", "Students", [2, 4, 5], {"schoolShort": ["Yale", "Yale", "Eureka"]}] ] }, "CHECK_CALL_COUNTS" : { @@ -1574,27 +1583,32 @@ ["ModifyColumn", "Students", "fullName", {"formula": "rec.lastName + ' - ' + rec.firstName"}], ["UpdateRecord", "_grist_Tables_column", 3, - {"formula": "rec.lastName + ' - ' + rec.firstName"}] + {"formula": "rec.lastName + ' - ' + rec.firstName"}], + ["BulkUpdateRecord", "Students", [1,2,3,4,5,6,8], { + "fullName": [ + "Obama - Barack", + "Bush - George W", + "Clinton - Bill", + "Bush - George H", + "Reagan - Ronald", + "Carter - Jimmy", + "Ford - Gerald" + ] + }], + ["BulkUpdateRecord", "Students", [1,2,3,4,5,6,8], { + "fullNameLen": [14,15,14,15,15,14,13] + }] ], "undo": [ ["ModifyColumn", "Students", "fullName", {"formula": "rec.firstName + ' ' + rec.lastName"}], ["UpdateRecord", "_grist_Tables_column", 3, - {"formula": "rec.firstName + ' ' + rec.lastName"}] - ], - "calc": [["BulkUpdateRecord", "Students", [1,2,3,4,5,6,8], { - "fullName": [ - "Obama - Barack", - "Bush - George W", - "Clinton - Bill", - "Bush - George H", - "Reagan - Ronald", - "Carter - Jimmy", - "Ford - Gerald" - ] - }], ["BulkUpdateRecord", "Students", [1,2,3,4,5,6,8], { - "fullNameLen": [14,15,14,15,15,14,13] - }]] + {"formula": "rec.firstName + ' ' + rec.lastName"}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"fullName": ["Barack Obama", "George W Bush", "Bill Clinton", "George H Bush", "Ronald Reagan", "Jimmy Carter", "Gerald Ford"]}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"fullNameLen": [12, 13, 12, 13, 13, 12, 11]}] + ] }, "CHECK_CALL_COUNTS" : { "Students" : { "fullName" : 7, "fullNameLen" : 7 } @@ -1611,17 +1625,16 @@ "stored": [ ["UpdateRecord", "Students", 2, {"firstName": "G.W."}], ["AddRecord", "Address", 11, {"country": "US"}], - ["UpdateRecord", "Address", 11, {"city": "Anytown"}] - - ], - "undo": [ - ["UpdateRecord", "Students", 2, {"firstName": "George W"}], - ["RemoveRecord", "Address", 11] - ], - "calc": [ + ["UpdateRecord", "Address", 11, {"city": "Anytown"}], ["UpdateRecord", "Address", 11, {"region": "North America" }], ["UpdateRecord", "Students", 2, {"fullName": "Bush - G.W."}], ["UpdateRecord", "Students", 2, {"fullNameLen": 11}] + ], + "undo": [ + ["UpdateRecord", "Students", 2, {"firstName": "George W"}], + ["RemoveRecord", "Address", 11], + ["UpdateRecord", "Students", 2, {"fullName": "Bush - George W"}], + ["UpdateRecord", "Students", 2, {"fullNameLen": 15}] ] }, "CHECK_CALL_COUNTS" : { @@ -1685,8 +1698,6 @@ "stored": [ ["ModifyColumn", "Students", "fullNameLen", {"isFormula": false, "type": "Int"}], - ["BulkUpdateRecord", "Students", [1,2,3,4,5,6,8], - {"fullNameLen": [14,11,14,15,15,14,13]}], ["UpdateRecord", "_grist_Tables_column", 4, {"isFormula": false, "type": "Int"}], ["ModifyColumn", "Students", "fullNameLen", @@ -1697,27 +1708,21 @@ ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], {"fullNameLen": [13, 10, 13, 14, 14, 13, 12]}], ["UpdateRecord", "_grist_Tables_column", 4, {"isFormula": false}] - ], "undo": [ ["ModifyColumn", "Students", "fullNameLen", {"isFormula": true, "type": "Any"}], ["UpdateRecord", "_grist_Tables_column", 4, {"isFormula": true, "type": "Any"}], - ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], - {"fullNameLen": [14, 11, 14, 15, 15, 14, 13]}], ["ModifyColumn", "Students", "fullNameLen", {"isFormula": false, "formula" : "len(rec.fullName)"}], ["UpdateRecord", "_grist_Tables_column", 4, {"isFormula": false, "formula" : "len(rec.fullName)"}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"fullNameLen": [14, 11, 14, 15, 15, 14, 13]}], ["ModifyColumn", "Students", "fullNameLen", {"isFormula": true}], ["UpdateRecord", "_grist_Tables_column", 4, {"isFormula": true}] - ] - // "calc": [ - // ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], - // {"fullNameLen": [13, 10, 13, 14, 14, 13, 12]}] - // ] }, "CHECK_CALL_COUNTS" : { "Students" : { "fullNameLen" : 7 } @@ -1838,20 +1843,17 @@ "ACTIONS" : { "stored" : [ ["ModifyColumn", "Address", "state", {"isFormula": true, "type": "Any"}], - ["UpdateRecord", "_grist_Tables_column", 27, {"isFormula": true, "type": "Any"}] + ["UpdateRecord", "_grist_Tables_column", 27, {"isFormula": true, "type": "Any"}], + ["BulkUpdateRecord", "Address", [2, 3, 4, 7, 10, 11], {"state": [null, null, null, null, null, null]}], + ["BulkUpdateRecord", "Students", [1, 2, 4, 5, 6, 8], {"schoolRegion": [null, null, null, null, null, null]}] ], "undo" : [ + ["ModifyColumn", "Address", "state", {"isFormula": false, "type": "Text"}], + ["UpdateRecord", "_grist_Tables_column", 27, {"isFormula": false, "type": "Text"}], ["BulkUpdateRecord", "Address", [2, 3, 4, 7, 10, 11], {"state": ["IL", "MD", "CT", "NY", "England", ""]}], - ["ModifyColumn", "Address", "state", {"isFormula": false, "type": "Text"}], - ["UpdateRecord", "_grist_Tables_column", 27, {"isFormula": false, "type": "Text"}] - - ], - - "calc" : [ - ["BulkUpdateRecord", "Address", [2, 3, 4, 7, 10, 11], {"state": [null, null, null, null, null, null]}], - ["BulkUpdateRecord", "Students", [1, 2, 4, 5, 6, 8], {"schoolRegion": [null, null, null, null, null, null]}] + ["BulkUpdateRecord", "Students", [1, 2, 4, 5, 6, 8], {"schoolRegion": ["NY", "CT", "CT", "IL", "MD", "CT"]}] ] }, "CHECK_CALL_COUNTS" : { @@ -1880,17 +1882,7 @@ ["ModifyColumn", "Students", "fullName", {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}], ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}], ["ModifyColumn", "Students", "schoolRegion", {"formula": "5*len($firstName) / $fullNameLen"}], - ["UpdateRecord", "_grist_Tables_column", 9, {"formula": "5*len($firstName) / $fullNameLen"}] - ], - "undo" : [ - ["ModifyColumn", "Students", "fullName", {"formula": "rec.lastName + ' - ' + rec.firstName"}], - ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "rec.lastName + ' - ' + rec.firstName"}], - ["ModifyColumn", "Students", "schoolRegion", - {"formula": "addr = $school.address\naddr.state if addr.country == 'US' else addr.region"}], - ["UpdateRecord", "_grist_Tables_column", 9, - {"formula": "addr = $school.address\naddr.state if addr.country == 'US' else addr.region"}] - ], - "calc" : [ + ["UpdateRecord", "_grist_Tables_column", 9, {"formula": "5*len($firstName) / $fullNameLen"}], ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], {"fullName" : [["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"], ["E","SyntaxError"]] @@ -1899,6 +1891,18 @@ // the calculation for that record fails with TypeError. ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], {"schoolRegion": [["E", "TypeError"], 2, 1, 2, 2, 1, 2]}] + ], + "undo" : [ + ["ModifyColumn", "Students", "fullName", {"formula": "rec.lastName + ' - ' + rec.firstName"}], + ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "rec.lastName + ' - ' + rec.firstName"}], + ["ModifyColumn", "Students", "schoolRegion", + {"formula": "addr = $school.address\naddr.state if addr.country == 'US' else addr.region"}], + ["UpdateRecord", "_grist_Tables_column", 9, + {"formula": "addr = $school.address\naddr.state if addr.country == 'US' else addr.region"}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"fullName": ["Obama - Barack", "Bush - G.W.", "Clinton - Bill", "Bush - George H", "Reagan - Ronald", "Carter - Jimmy", "Ford - Gerald"]}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"schoolRegion": [null, null, "Europe", null, null, null, null]}] ] }, "CHECK_CALL_COUNTS" : { @@ -1914,15 +1918,15 @@ "ACTIONS" : { "stored" : [ ["ModifyColumn", "Students", "fullName", {"formula": "$firstName"}], - ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "$firstName"}] + ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "$firstName"}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"fullName": ["Barack", "G.W.", "Bill", "George H", "Ronald", "Jimmy", "Gerald"]}] ], "undo" : [ ["ModifyColumn", "Students", "fullName", {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}], - ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}] - ], - "calc" : [ + ["UpdateRecord", "_grist_Tables_column", 3, {"formula": "!#@%&T#$UDSAIKVFsdhifzsk"}], ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], - {"fullName": ["Barack", "G.W.", "Bill", "George H", "Ronald", "Jimmy", "Gerald"]}] + {"fullName": [["E", "SyntaxError"], ["E", "SyntaxError"], ["E", "SyntaxError"], ["E", "SyntaxError"], ["E", "SyntaxError"], ["E", "SyntaxError"], ["E", "SyntaxError"]]}] ] } }], @@ -1987,7 +1991,7 @@ }, { //---------------------------------------------------------------------- - "TEST_CASE": "column_conversions: DateTime->Int->Numeric->DateTime", + "TEST_CASE": "column_conversions", //---------------------------------------------------------------------- "BODY": [ ["LOAD_SAMPLE", "simplest"], @@ -2164,7 +2168,8 @@ ["UpdateRecord", "_grist_Tables", 5, {"primaryViewId": 2}], ["AddRecord", "Bar", 1, {"foo": 0, "hello": "a", "manualSort": 1.0}], ["AddRecord", "Bar", 2, {"foo": 1, "hello": "b", "manualSort": 2.0}], - ["AddRecord", "Bar", 3, {"foo": 1, "hello": "c", "manualSort": 3.0}] + ["AddRecord", "Bar", 3, {"foo": 1, "hello": "c", "manualSort": 3.0}], + ["BulkUpdateRecord", "Bar", [1, 2, 3], {"world": ["A", "B", "C"]}] ], "undo": [ ["RemoveTable", "Foo"], @@ -2188,9 +2193,6 @@ ["RemoveRecord", "Bar", 2], ["RemoveRecord", "Bar", 3] ], - "calc": [ - ["BulkUpdateRecord", "Bar", [1, 2, 3], {"world": ["A", "B", "C"]}] - ], "retValue": [ // AddTable "Foo" retValue { @@ -2260,7 +2262,6 @@ // As part of adding a table, we also set the primaryViewId. ["UpdateRecord", "_grist_Tables", 4, {"primaryViewId": 1}] ], - "calc": [], "undo": [ ["RemoveTable", "Foo"], ["RemoveRecord", "_grist_Tables", 4], @@ -2323,7 +2324,18 @@ ["RemoveColumn", "Students", "school"], ["BulkRemoveRecord", "_grist_Tables_column", [10, 12]], ["RemoveRecord", "_grist_Tables", 2], - ["RemoveTable", "Schools"] + ["RemoveTable", "Schools"], + // Assert that calc actions include a RemoveColumn for Students.school + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"schoolRegion": [["E","AttributeError"], ["E","AttributeError"], + ["E","AttributeError"], ["E","AttributeError"], ["E","AttributeError"], + ["E","AttributeError"], ["E","AttributeError"]] + }], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"schoolShort": [["E","AttributeError"], ["E","AttributeError"], + ["E","AttributeError"], ["E","AttributeError"], ["E","AttributeError"], + ["E","AttributeError"], ["E","AttributeError"]] + }] ], "undo": [ ["AddRecord", "_grist_Tables_column", 5, @@ -2346,20 +2358,9 @@ ["AddTable", "Schools", [ {"isFormula": false, "formula": "", "type": "Text", "id": "name"}, {"isFormula": false, "formula": "", "type": "Ref:Address", "id": "address"}] - ] - ], - "calc": [ - // Assert that calc actions include a RemoveColumn for Students.school - ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], - {"schoolRegion": [["E","AttributeError"], ["E","AttributeError"], - ["E","AttributeError"], ["E","AttributeError"], ["E","AttributeError"], - ["E","AttributeError"], ["E","AttributeError"]] - }], - ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], - {"schoolShort": [["E","AttributeError"], ["E","AttributeError"], - ["E","AttributeError"], ["E","AttributeError"], ["E","AttributeError"], - ["E","AttributeError"], ["E","AttributeError"]] - }] + ], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], {"schoolRegion": ["NY", "CT", "Europe", "CT", "IL", "MD", "CT"]}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], {"schoolShort": ["Columbia", "Yale", "Oxford", "Yale", "Eureka", "U.S.", "Yale"]}] ] }, "CHECK_CALL_COUNTS" : { @@ -2408,8 +2409,14 @@ }], ["AddRecord", "_grist_Tables", 1, {"tableId": "Students"}], ["BulkAddRecord", "Students", [1, 2, 3, 4, 5, 6, 8], { + "firstName": ["Barack", "George W", "Bill", "George H", "Ronald", "Jimmy", "Gerald"], + "fullName": ["Barack Obama", "George W Bush", "Bill Clinton", "George H Bush", "Ronald Reagan", "Jimmy Carter", "Gerald Ford"], + "fullNameLen": [12, 13, 12, 13, 13, 12, 11], "lastName": ["Obama", "Bush", "Clinton", "Bush", "Reagan", "Carter", "Ford"], - "firstName": ["Barack", "George W", "Bill", "George H", "Ronald", "Jimmy", "Gerald"] + "schoolRegion": [["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"], + ["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"]], + "schoolShort": [["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"], + ["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"]] }], ["AddTable", "Students", [ { "isFormula": false, "formula": "", "type": "Text", "id": "firstName"}, @@ -2469,19 +2476,6 @@ ["UpdateRecord", "_grist_Tables", 2, {"tableId": "Schools"}], ["ModifyColumn", "People", "school", {"type": "Int"}], ["UpdateRecord", "_grist_Tables_column", 5, {"type": "Ref:Schools"}] - ], - "calc": [ - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"fullName": [ - "Barack Obama", "George W Bush", "Bill Clinton", "George H Bush", - "Ronald Reagan", "Jimmy Carter", "Gerald Ford"] - }], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"fullNameLen": [12, 13, 12, 13, 13, 12, 11]}], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"schoolRegion": ["NY", "CT", "Europe", "CT", "IL", "MD", "CT"]}], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"schoolShort": ["Columbia", "Yale", "Oxford", "Yale", "Eureka", "U.S.", "Yale"]}] ] }, "CHECK_CALL_COUNTS" : { @@ -2508,19 +2502,6 @@ ["UpdateRecord", "_grist_Tables", 1, {"tableId": "People"}], ["RenameTable", "People", "PEOPLE"], ["UpdateRecord", "_grist_Tables", 1, {"tableId": "PEOPLE"}] - ], - "calc": [ - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"fullName": [ - "Barack Obama", "George W Bush", "Bill Clinton", "George H Bush", - "Ronald Reagan", "Jimmy Carter", "Gerald Ford"] - }], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"fullNameLen": [12, 13, 12, 13, 13, 12, 11]}], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"schoolRegion": ["NY", "CT", "Europe", "CT", "IL", "MD", "CT"]}], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"schoolShort": ["Columbia", "Yale", "Oxford", "Yale", "Eureka", "U.S.", "Yale"]}] ] } }], @@ -2531,13 +2512,16 @@ "ACTIONS": { "stored": [ ["RemoveRecord", "School", 8], - ["BulkUpdateRecord", "People", [2, 4], {"school": [0, 0]}]], + ["BulkUpdateRecord", "People", [2, 4], {"school": [0, 0]}], + ["BulkUpdateRecord", "People", [2, 4], {"schoolRegion": [null, null]}], + ["BulkUpdateRecord", "People", [2, 4], {"schoolShort": ["", ""]}] + ], "undo": [ ["AddRecord", "School", 8, {"name": "Yale University", "address": 4}], - ["BulkUpdateRecord", "People", [2, 4], {"school": [8, 8]}]], - "calc": [ - ["BulkUpdateRecord", "People", [2, 4], {"schoolRegion": [null, null]}], - ["BulkUpdateRecord", "People", [2, 4], {"schoolShort": ["", ""]}]] + ["BulkUpdateRecord", "People", [2, 4], {"school": [8, 8]}], + ["BulkUpdateRecord", "People", [2, 4], {"schoolRegion": ["CT", "CT"]}], + ["BulkUpdateRecord", "People", [2, 4], {"schoolShort": ["Yale", "Yale"]}] + ] }, "CHECK_CALL_COUNTS" : { "People" : { "schoolRegion" : 2, "schoolShort" : 2 } @@ -2603,19 +2587,6 @@ ["UpdateRecord", "_grist_Tables", 2, {"tableId": "Schools"}], ["ModifyColumn", "People", "school", {"type": "Int"}], ["UpdateRecord", "_grist_Tables_column", 5, {"type": "Ref:Schools"}] - ], - "calc": [ - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"fullName": [ - "Barack Obama", "George W Bush", "Bill Clinton", "George H Bush", - "Ronald Reagan", "Jimmy Carter", "Gerald Ford"] - }], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"fullNameLen": [12, 13, 12, 13, 13, 12, 11]}], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"schoolRegion": ["NY", "CT", "Europe", "CT", "IL", "MD", "CT"]}], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5, 6, 8], - {"schoolShort": ["Columbia", "Yale", "Oxford", "Yale", "Eureka", "U.S.", "Yale"]}] ] }, "CHECK_CALL_COUNTS" : { @@ -2629,13 +2600,16 @@ "ACTIONS": { "stored": [ ["RemoveRecord", "School", 8], - ["BulkUpdateRecord", "People", [2, 4], {"school": [0, 0]}]], + ["BulkUpdateRecord", "People", [2, 4], {"school": [0, 0]}], + ["BulkUpdateRecord", "People", [2, 4], {"schoolRegion": [null, null]}], + ["BulkUpdateRecord", "People", [2, 4], {"schoolShort": ["", ""]}] + ], "undo": [ ["AddRecord", "School", 8, {"name": "Yale University", "address": 4}], - ["BulkUpdateRecord", "People", [2, 4], {"school": [8, 8]}]], - "calc": [ - ["BulkUpdateRecord", "People", [2, 4], {"schoolRegion": [null, null]}], - ["BulkUpdateRecord", "People", [2, 4], {"schoolShort": ["", ""]}]] + ["BulkUpdateRecord", "People", [2, 4], {"school": [8, 8]}], + ["BulkUpdateRecord", "People", [2, 4], {"schoolRegion": ["CT", "CT"]}], + ["BulkUpdateRecord", "People", [2, 4], {"schoolShort": ["Yale", "Yale"]}] + ] }, "CHECK_CALL_COUNTS" : { "People" : { "schoolRegion" : 2, "schoolShort" : 2 } @@ -2749,7 +2723,15 @@ "errorText": "", "outputText": "[2000000.0, AltText('George W'), AltText('Bill'), AltText('George H'), AltText('Ronald'), AltText('Jimmy'), AltText('Gerald')]\n"}], ["UpdateRecord", "_grist_REPL_Hist", 6, - {"code": "len(Schools.all)", "errorText": "", "outputText": "6\n"}] + {"code": "len(Schools.all)", "errorText": "", "outputText": "6\n"}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], { + "fullName": [["E", "TypeError"], ["E","TypeError"], ["E","TypeError"], + ["E","TypeError"], ["E","TypeError"], ["E","TypeError"], ["E","TypeError"]] + }], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], { + "fullNameLen": [["E","TypeError"], ["E","TypeError"], ["E","TypeError"], + ["E","TypeError"], ["E","TypeError"], ["E","TypeError"], ["E","TypeError"]] + }] ], "undo": [ ["RemoveRecord", "_grist_REPL_Hist", 6], @@ -2763,17 +2745,11 @@ "errorText": "", "outputText": "['Barack', 'George W', 'Bill', 'George H', 'Ronald', 'Jimmy', 'Gerald']\n"}], ["UpdateRecord", "_grist_REPL_Hist", 6, {"code": "list(Students.all.firstName)", "errorText": "", - "outputText": "[2000000.0, AltText('George W'), AltText('Bill'), AltText('George H'), AltText('Ronald'), AltText('Jimmy'), AltText('Gerald')]\n"}] - ], - "calc" : [ - ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], { - "fullName": [["E", "TypeError"], ["E","TypeError"], ["E","TypeError"], - ["E","TypeError"], ["E","TypeError"], ["E","TypeError"], ["E","TypeError"]] - }], - ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], { - "fullNameLen": [["E","TypeError"], ["E","TypeError"], ["E","TypeError"], - ["E","TypeError"], ["E","TypeError"], ["E","TypeError"], ["E","TypeError"]] - }] + "outputText": "[2000000.0, AltText('George W'), AltText('Bill'), AltText('George H'), AltText('Ronald'), AltText('Jimmy'), AltText('Gerald')]\n"}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"fullName": ["Barack Obama", "George W Bush", "Bill Clinton", "George H Bush", "Ronald Reagan", "Jimmy Carter", "Gerald Ford"]}], + ["BulkUpdateRecord", "Students", [1, 2, 3, 4, 5, 6, 8], + {"fullNameLen": [12, 13, 12, 13, 13, 12, 11]}] ], "retValue": [true,null,null,true,true] } diff --git a/sandbox/grist/useractions.py b/sandbox/grist/useractions.py index 416ebc0e..5e3f2f4a 100644 --- a/sandbox/grist/useractions.py +++ b/sandbox/grist/useractions.py @@ -8,6 +8,7 @@ import acl import actions import column import identifiers +from objtypes import strict_equal import schema import summary import import_actions @@ -1153,12 +1154,8 @@ class UserActions(object): return if from_formula and not to_formula: - # Make sure the old column is up to date, in case anything was to be recomputed. This - # creates calc actions for a formula column, which we should discard in case we are - # converting formula to non-formula. - calc_actions_len = len(self._engine.out_actions.calc) + # Make sure the old column is up to date, in case anything was to be recomputed. self._engine.bring_col_up_to_date(old_column) - del self._engine.out_actions.calc[calc_actions_len:] # Get the values from the old column, which is about to be destroyed. all_rows = list(table.row_ids) @@ -1177,39 +1174,33 @@ class UserActions(object): # Fill in the new column by converting the values from the old column. If the type hasn't # changed, or is compatible, the conversion should return the value unchanged. - changed_rows = [] + changes = [] for row_id in all_rows: orig_value = all_old_values[row_id] new_value = new_column.convert(orig_value) - if not usertypes.strict_equal(orig_value, new_value): + if not strict_equal(orig_value, new_value): new_column.set(row_id, new_value) - changed_rows.append(row_id) + changes.append((row_id, orig_value, new_column.raw_get(row_id))) - def make_bulk(rows, get_value): - return actions.BulkUpdateRecord(table_id, rows, {col_id: map(get_value, rows)}).simplify() - - if not from_formula: - undo_action = make_bulk(all_rows if to_formula else changed_rows, all_old_values.get) - if undo_action: - # The UNDO action needs to be inserted before the one created by ModifyColumn, so that on - # undo, we apply ModifyColumn first (getting the correct type), then set the values of - # that type. (The insert(-1) feels hacky, but I see no better way of doing it.) - assert isinstance(self._engine.out_actions.undo[-1], actions.ModifyColumn), \ - "ModifyColumn not where expected in undo list" - self._engine.out_actions.undo.insert(-1, undo_action) + # Prepare the changes as if for a formula column; they'd get merged at this point with any + # previous calc_changes for this column. + if changes: + self._engine.out_actions.summary.add_changes(table_id, col_id, changes) if not to_formula: - stored_action = make_bulk(all_rows if from_formula else changed_rows, new_column.raw_get) - if stored_action: - # Produce the change update. The sandbox is already correct, so we don't need to apply it. - # (Also, it would be wrong to apply via docactions, since that would generate an "undo" - # in the wrong order relative to the ModifyColumn's undo.) - self._engine.out_actions.stored.append(stored_action) + # If converting to non-formula, any previously prepared calc actions should be removed from + # calc summary and actualized now (so that they don't override subsequent changes). - if to_formula: - calc_action = make_bulk(changed_rows, new_column.raw_get) - if calc_action: - self._engine.out_actions.calc.append(calc_action) + # The UNDO action needs to be inserted before the one created by ModifyColumn, so that on + # undo, we apply ModifyColumn first (getting the correct type), then set the values of + # that type. We do it by moving the last (ModifyColumn) action to the end. + assert isinstance(self._engine.out_actions.undo[-1], actions.ModifyColumn), \ + "ModifyColumn not where expected in undo list" + mod_action = self._engine.out_actions.undo.pop() + try: + self._engine.out_actions.flush_calc_changes_for_column(table_id, col_id) + finally: + self._engine.out_actions.undo.append(mod_action) @useraction diff --git a/sandbox/grist/usertypes.py b/sandbox/grist/usertypes.py index 8d4c6c5a..e80f4cc5 100644 --- a/sandbox/grist/usertypes.py +++ b/sandbox/grist/usertypes.py @@ -22,11 +22,6 @@ log = logger.Logger(__name__, logger.INFO) NoneType = type(None) -def strict_equal(a, b): - """Checks the equality of the types of the values as well as the values.""" - # pylint: disable=unidiomatic-typecheck - return type(a) == type(b) and a == b - # Note that this matches the defaults in app/common/gristTypes.js _type_defaults = { 'Any': None,