Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) New type conversion in the backend

Browse Source 
Summary: This is https://phab.getgrist.com/D3205 plus some changes (https://github.com/dsagal/grist/compare/type-convert...type-convert-server?expand=1) that move the conversion process to the backend. A new user action ConvertFromColumn uses `call_external` so that the data engine can delegate back to ActiveDoc. Code for creating formatters and parsers is significantly refactored so that most of the logic is in `common` and can be used in different ways.

Test Plan: The original diff adds plenty of tests.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Subscribers: dsagal

Differential Revision: https://phab.getgrist.com/D3240
This commit is contained in:
Alex Hall
2022-02-04 13:13:03 +02:00
parent 4890a1fe89
commit 5d671bf0b3
25 changed files with 593 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
app/client/components/ColumnTransform.ts
View File

@@ -142,6 +142,10 @@ export class ColumnTransform extends Disposable {
return actions.every(action => (
// ['AddColumn', USER_TABLE, 'gristHelper_Transform', colInfo]
(action[2] === 'gristHelper_Transform') ||
// ['AddColumn', USER_TABLE, 'gristHelper_Converted', colInfo]
(action[2] === 'gristHelper_Converted') ||
// ['ConvertFromColumn', USER_TABLE, SOURCE_COLUMN, 'gristHelper_Converted']
(action[3] === 'gristHelper_Converted') ||
// ["SetDisplayFormula", USER_TABLE, ...]
(action[0] === 'SetDisplayFormula') ||
// ['UpdateRecord', '_grist_Table_column', transformColId, ...]
@@ -192,7 +196,6 @@ export class ColumnTransform extends Disposable {
// Define variables used after await, since this will be disposed by then.
const transformColId = this.transformColumn.colId();
const field = this.field;
const fieldBuilder = this._fieldBuilder;
const origRef = this.origColumn.getRowId();
const tableData = this._tableData;
this.isCallPending(true);
@@ -201,17 +204,36 @@ export class ColumnTransform extends Disposable {
// TODO: Values flicker during executing since transform column remains a formula as values are copied
// back to the original column. The CopyFromColumn useraction really ought to be "CopyAndRemove" since
// that seems the best way to avoid calculating the formula on wrong values.
return await tableData.sendTableAction(['CopyFromColumn', transformColId, this.origColumn.colId(),
JSON.stringify(fieldBuilder.options())]);
await this.gristDoc.docData.sendActions(this.executeActions());
}
} finally {
// Wait until the change completed to set column back, to avoid value flickering.
field.colRef(origRef);
void tableData.sendTableAction(['RemoveColumn', transformColId]);
this.cleanup();
this.dispose();
}
}
/**
* The user actions to send when actually executing the transform.
*/
protected executeActions(): UserAction[] {
return [
[
'CopyFromColumn',
this._tableData.tableId,
this.transformColumn.colId(),
this.origColumn.colId(),
JSON.stringify(this._fieldBuilder.options()),
],
];
}
protected cleanup() {
// For overriding
}
protected getIdentityFormula() {
return 'return $' + this.origColumn.colId();
}

103
app/client/components/TypeConversion.ts
View File

@@ -6,7 +6,6 @@
import {DocModel} from 'app/client/models/DocModel';
import {ColumnRec} from 'app/client/models/entities/ColumnRec';
import * as UserType from 'app/client/widgets/UserType';
import * as gristTypes from 'app/common/gristTypes';
import {isFullReferencingType} from 'app/common/gristTypes';
import * as gutil from 'app/common/gutil';
@@ -90,7 +89,7 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
type: addColTypeSuffix(toTypeMaybeFull, origCol, docModel),
isFormula: true,
visibleCol: 0,
formula: "", // Will be filled in at the end.
formula: "CURRENT_CONVERSION(rec)",
};
const prevOptions = origCol.widgetOptionsJson.peek() || {};
@@ -139,23 +138,35 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
case 'RefList':
{
// Set suggested destination table and visible column.
// Null if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen).
const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`)!;
// Undefined if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen).
const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`) || undefined;
// Finds a reference suggestion column and sets it as the current reference value.
const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100);
if (!columnData) { break; }
columnData.delete(gristTypes.getDefaultForType(origCol.type()));
let suggestedColRef: number;
let suggestedTableId: string;
const origColTypeInfo = gristTypes.extractInfoFromColType(origCol.type.peek());
if (!optTableId && origColTypeInfo.type === "Ref" || origColTypeInfo.type === "RefList") {
// When converting between Ref and Reflist, initially suggest the same table and visible column.
// When converting, if the table is the same, it's a special case.
// The visible column will not affect conversion.
// It will simply wrap the reference (row ID) in a list or extract the one element of a reference list.
suggestedColRef = origCol.visibleCol.peek();
suggestedTableId = origColTypeInfo.tableId;
} else {
// Finds a reference suggestion column and sets it as the current reference value.
const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100);
if (!columnData) { break; }
columnData.delete(gristTypes.getDefaultForType(origCol.type()));
// 'findColFromValues' function requires an array since it sends the values to the sandbox.
const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId);
const suggestedColRef = matches.find(match => match !== origCol.getRowId());
if (!suggestedColRef) { break; }
const suggestedCol = docModel.columns.getRowModel(suggestedColRef);
const suggestedTableId = suggestedCol.table().tableId();
if (optTableId && suggestedTableId !== optTableId) {
console.warn("Inappropriate column received from findColFromValues");
break;
// 'findColFromValues' function requires an array since it sends the values to the sandbox.
const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId);
suggestedColRef = matches.find(match => match !== origCol.getRowId())!;
if (!suggestedColRef) { break; }
const suggestedCol = docModel.columns.getRowModel(suggestedColRef);
suggestedTableId = suggestedCol.table().tableId();
if (optTableId && suggestedTableId !== optTableId) {
console.warn("Inappropriate column received from findColFromValues");
break;
}
}
colInfo.type = `${toType}:${suggestedTableId}`;
colInfo.visibleCol = suggestedColRef;
@@ -163,11 +174,9 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
}
}
const newOptions = UserType.mergeOptions(widgetOptions || {}, colInfo.type);
if (widgetOptions) {
colInfo.widgetOptions = JSON.stringify(widgetOptions);
}
colInfo.formula = getDefaultFormula(docModel, origCol, colInfo.type, colInfo.visibleCol, newOptions);
return colInfo;
}
@@ -184,62 +193,6 @@ export async function setDisplayFormula(
}
}
// Given the original column and info about the new column, returns the formula to use for the
// transform column to do the transformation.
export function getDefaultFormula(
docModel: DocModel, origCol: ColumnRec, newType: string,
newVisibleCol: number, newWidgetOptions: any): string {
const colId = origCol.colId();
const oldVisibleColName = isReferenceCol(origCol) ?
getVisibleColName(docModel, origCol.visibleCol()) : undefined;
let origValFormula = oldVisibleColName ?
// The `str()` below converts AltText to plain text.
`($${colId}.${oldVisibleColName}
if ISREF($${colId}) or ISREFLIST($${colId})
else str($${colId}))`
: `$${colId}`;
if (origCol.type.peek() === 'ChoiceList') {
origValFormula = `grist.ChoiceList.toString($${colId})`;
}
const toTypePure: string = gristTypes.extractTypeFromColType(newType);
// The args are used to construct the call to grist.TYPE.typeConvert(value, [params]).
// Optional parameters depend on the type; see sandbox/grist/usertypes.py
const args: string[] = [origValFormula];
switch (toTypePure) {
case 'Ref':
case 'RefList':
{
const table = gutil.removePrefix(newType, toTypePure + ":");
args.push(table || 'None');
const visibleColName = getVisibleColName(docModel, newVisibleCol);
if (visibleColName) {
args.push(q(visibleColName));
}
break;
}
case 'Date': {
args.push(q(newWidgetOptions.dateFormat));
break;
}
case 'DateTime': {
const timezone = gutil.removePrefix(newType, "DateTime:") || '';
const format = newWidgetOptions.dateFormat + ' ' + newWidgetOptions.timeFormat;
args.push(q(format), q(timezone));
break;
}
}
return `grist.${gristTypes.getGristType(toTypePure)}.typeConvert(${args.join(', ')})`;
}
function q(value: string): string {
return "'" + value.replace(/'/g, "\\'") + "'";
}
// Returns the name of the visibleCol given its rowId.
function getVisibleColName(docModel: DocModel, visibleColRef: number): string|undefined {
return visibleColRef ? docModel.columns.getRowModel(visibleColRef).colId() : undefined;

72
app/client/components/TypeTransform.ts
View File

@@ -10,13 +10,12 @@ import {ColumnTransform} from 'app/client/components/ColumnTransform';
import {GristDoc} from 'app/client/components/GristDoc';
import * as TypeConversion from 'app/client/components/TypeConversion';
import {reportError} from 'app/client/models/errors';
import * as modelUtil from 'app/client/models/modelUtil';
import {cssButtonRow} from 'app/client/ui/RightPanel';
import {basicButton, primaryButton} from 'app/client/ui2018/buttons';
import {testId} from 'app/client/ui2018/cssVars';
import {FieldBuilder} from 'app/client/widgets/FieldBuilder';
import {NewAbstractWidget} from 'app/client/widgets/NewAbstractWidget';
import {ColValues} from 'app/common/DocActions';
import {ColValues, UserAction} from 'app/common/DocActions';
import {Computed, dom, fromKo, Observable} from 'grainjs';
import isEmpty = require('lodash/isEmpty');
import pickBy = require('lodash/pickBy');
@@ -86,15 +85,6 @@ export class TypeTransform extends ColumnTransform {
);
}
protected async resetToDefaultFormula() {
if (!this.isFinalizing()) {
const toType = this.transformColumn.type.peek();
const formula = TypeConversion.getDefaultFormula(this.gristDoc.docModel, this.origColumn,
toType, this.field.visibleColRef(), this.field.widgetOptionsJson());
await modelUtil.setSaveValue(this.transformColumn.formula, formula);
}
}
/**
* Overrides parent method to initialize the transform column with guesses as to the particular
* type and column options.
@@ -103,20 +93,55 @@ export class TypeTransform extends ColumnTransform {
protected async addTransformColumn(toType: string) {
const docModel = this.gristDoc.docModel;
const colInfo = await TypeConversion.prepTransformColInfo(docModel, this.origColumn, this.origDisplayCol, toType);
const newColInfo = await this._tableData.sendTableAction(['AddColumn', 'gristHelper_Transform', colInfo]);
const tcol = docModel.columns.getRowModel(newColInfo.colRef);
await TypeConversion.setDisplayFormula(docModel, tcol);
return newColInfo.colRef;
const newColInfos = await this._tableData.sendTableActions([
['AddColumn', 'gristHelper_Converted', {...colInfo, isFormula: false, formula: ''}],
['AddColumn', 'gristHelper_Transform', colInfo],
]);
const transformColRef = newColInfos[1].colRef;
this.transformColumn = docModel.columns.getRowModel(transformColRef);
await this.convertValues();
return transformColRef;
}
protected convertValuesActions(): UserAction[] {
const tableId = this._tableData.tableId;
const srcColId = this.origColumn.colId.peek();
const dstColId = "gristHelper_Converted";
const type = this.transformColumn.type.peek();
const widgetOptions = this.transformColumn.widgetOptions.peek();
const visibleColRef = this.transformColumn.visibleCol.peek();
return [['ConvertFromColumn', tableId, srcColId, dstColId, type, widgetOptions, visibleColRef]];
}
protected async convertValues() {
await Promise.all([
this.gristDoc.docData.sendActions(this.convertValuesActions()),
TypeConversion.setDisplayFormula(this.gristDoc.docModel, this.transformColumn),
]);
}
protected executeActions(): UserAction[] {
return [...this.convertValuesActions(), ...super.executeActions()];
}
/**
* Overrides parent method to subscribe to changes to the transform column.
*/
protected postAddTransformColumn() {
// When a user-initiated change is saved to type or widgetOptions, update the formula.
this.autoDispose(this.transformColumn.type.subscribe(this.resetToDefaultFormula, this, "save"));
this.autoDispose(this.transformColumn.visibleCol.subscribe(this.resetToDefaultFormula, this, "save"));
this.autoDispose(this.field.widgetOptionsJson.subscribe(this.resetToDefaultFormula, this, "save"));
// When a user-initiated change is saved to type or widgetOptions, reconvert the values
// Need to subscribe to both 'change' and 'save' for type which can come from setting the type itself
// or e.g. a change to DateTime timezone.
this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "change"));
this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "save"));
this.autoDispose(this.transformColumn.visibleCol.subscribe(this.convertValues, this, "save"));
this.autoDispose(this.field.widgetOptionsJson.subscribe(this.convertValues, this, "save"));
}
/**
* Overrides parent method to delete extra column
*/
protected cleanup() {
void this._tableData.sendTableAction(['RemoveColumn', 'gristHelper_Converted']);
}
/**
@@ -129,9 +154,10 @@ export class TypeTransform extends ColumnTransform {
const tcol = this.transformColumn;
const changedInfo = pickBy(colInfo, (val, key) =>
(val !== tcol[key as keyof TypeConversion.ColInfo].peek()));
return Promise.all([
isEmpty(changedInfo) ? undefined : tcol.updateColValues(changedInfo as ColValues),
TypeConversion.setDisplayFormula(docModel, tcol, changedInfo.visibleCol)
]);
if (!isEmpty(changedInfo)) {
// Update the transform column, particularly the type.
// This will trigger the subscription in postAddTransformColumn and lead to calling convertValues.
await tcol.updateColValues(changedInfo as ColValues);
}
}
}