Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

(core) New type conversion in the backend

Browse Source 
Summary: This is https://phab.getgrist.com/D3205 plus some changes (https://github.com/dsagal/grist/compare/type-convert...type-convert-server?expand=1) that move the conversion process to the backend. A new user action ConvertFromColumn uses `call_external` so that the data engine can delegate back to ActiveDoc. Code for creating formatters and parsers is significantly refactored so that most of the logic is in `common` and can be used in different ways.

Test Plan: The original diff adds plenty of tests.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Subscribers: dsagal

Differential Revision: https://phab.getgrist.com/D3240
This commit is contained in:
Alex Hall 2022-02-04 13:13:03 +02:00
parent 4890a1fe89
commit 5d671bf0b3
25 changed files with 593 additions and 492 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
app/client/components/ColumnTransform.ts
View File

@ -142,6 +142,10 @@ export class ColumnTransform extends Disposable {
return actions.every(action => ( return actions.every(action => (
// ['AddColumn', USER_TABLE, 'gristHelper_Transform', colInfo] // ['AddColumn', USER_TABLE, 'gristHelper_Transform', colInfo]
(action[2] === 'gristHelper_Transform') || (action[2] === 'gristHelper_Transform') ||
// ['AddColumn', USER_TABLE, 'gristHelper_Converted', colInfo]
(action[2] === 'gristHelper_Converted') ||
// ['ConvertFromColumn', USER_TABLE, SOURCE_COLUMN, 'gristHelper_Converted']
(action[3] === 'gristHelper_Converted') ||
// ["SetDisplayFormula", USER_TABLE, ...] // ["SetDisplayFormula", USER_TABLE, ...]
(action[0] === 'SetDisplayFormula') || (action[0] === 'SetDisplayFormula') ||
// ['UpdateRecord', '_grist_Table_column', transformColId, ...] // ['UpdateRecord', '_grist_Table_column', transformColId, ...]
@ -192,7 +196,6 @@ export class ColumnTransform extends Disposable {
// Define variables used after await, since this will be disposed by then. // Define variables used after await, since this will be disposed by then.
const transformColId = this.transformColumn.colId(); const transformColId = this.transformColumn.colId();
const field = this.field; const field = this.field;
const fieldBuilder = this._fieldBuilder;
const origRef = this.origColumn.getRowId(); const origRef = this.origColumn.getRowId();
const tableData = this._tableData; const tableData = this._tableData;
this.isCallPending(true); this.isCallPending(true);
@ -201,17 +204,36 @@ export class ColumnTransform extends Disposable {
// TODO: Values flicker during executing since transform column remains a formula as values are copied // TODO: Values flicker during executing since transform column remains a formula as values are copied
// back to the original column. The CopyFromColumn useraction really ought to be "CopyAndRemove" since // back to the original column. The CopyFromColumn useraction really ought to be "CopyAndRemove" since
// that seems the best way to avoid calculating the formula on wrong values. // that seems the best way to avoid calculating the formula on wrong values.
return await tableData.sendTableAction(['CopyFromColumn', transformColId, this.origColumn.colId(), await this.gristDoc.docData.sendActions(this.executeActions());
JSON.stringify(fieldBuilder.options())]);
} }
} finally { } finally {
// Wait until the change completed to set column back, to avoid value flickering. // Wait until the change completed to set column back, to avoid value flickering.
field.colRef(origRef); field.colRef(origRef);
void tableData.sendTableAction(['RemoveColumn', transformColId]); void tableData.sendTableAction(['RemoveColumn', transformColId]);
this.cleanup();
this.dispose(); this.dispose();
} }
} }
/**
* The user actions to send when actually executing the transform.
*/
protected executeActions(): UserAction[] {
return [
[
'CopyFromColumn',
this._tableData.tableId,
this.transformColumn.colId(),
this.origColumn.colId(),
JSON.stringify(this._fieldBuilder.options()),
],
];
}
protected cleanup() {
// For overriding
}
protected getIdentityFormula() { protected getIdentityFormula() {
return 'return $' + this.origColumn.colId(); return 'return $' + this.origColumn.colId();
} }

103
app/client/components/TypeConversion.ts
View File

@ -6,7 +6,6 @@
import {DocModel} from 'app/client/models/DocModel'; import {DocModel} from 'app/client/models/DocModel';
import {ColumnRec} from 'app/client/models/entities/ColumnRec'; import {ColumnRec} from 'app/client/models/entities/ColumnRec';
import * as UserType from 'app/client/widgets/UserType';
import * as gristTypes from 'app/common/gristTypes'; import * as gristTypes from 'app/common/gristTypes';
import {isFullReferencingType} from 'app/common/gristTypes'; import {isFullReferencingType} from 'app/common/gristTypes';
import * as gutil from 'app/common/gutil'; import * as gutil from 'app/common/gutil';
@ -90,7 +89,7 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
type: addColTypeSuffix(toTypeMaybeFull, origCol, docModel), type: addColTypeSuffix(toTypeMaybeFull, origCol, docModel),
isFormula: true, isFormula: true,
visibleCol: 0, visibleCol: 0,
formula: "", // Will be filled in at the end. formula: "CURRENT_CONVERSION(rec)",
}; };
const prevOptions = origCol.widgetOptionsJson.peek() || {}; const prevOptions = origCol.widgetOptionsJson.peek() || {};
@ -139,23 +138,35 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
case 'RefList': case 'RefList':
{ {
// Set suggested destination table and visible column. // Set suggested destination table and visible column.
// Null if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen). // Undefined if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen).
const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`)!; const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`) || undefined;
// Finds a reference suggestion column and sets it as the current reference value. let suggestedColRef: number;
const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100); let suggestedTableId: string;
if (!columnData) { break; } const origColTypeInfo = gristTypes.extractInfoFromColType(origCol.type.peek());
columnData.delete(gristTypes.getDefaultForType(origCol.type())); if (!optTableId && origColTypeInfo.type === "Ref" || origColTypeInfo.type === "RefList") {
// When converting between Ref and Reflist, initially suggest the same table and visible column.
// When converting, if the table is the same, it's a special case.
// The visible column will not affect conversion.
// It will simply wrap the reference (row ID) in a list or extract the one element of a reference list.
suggestedColRef = origCol.visibleCol.peek();
suggestedTableId = origColTypeInfo.tableId;
} else {
// Finds a reference suggestion column and sets it as the current reference value.
const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100);
if (!columnData) { break; }
columnData.delete(gristTypes.getDefaultForType(origCol.type()));
// 'findColFromValues' function requires an array since it sends the values to the sandbox. // 'findColFromValues' function requires an array since it sends the values to the sandbox.
const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId); const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId);
const suggestedColRef = matches.find(match => match !== origCol.getRowId()); suggestedColRef = matches.find(match => match !== origCol.getRowId())!;
if (!suggestedColRef) { break; } if (!suggestedColRef) { break; }
const suggestedCol = docModel.columns.getRowModel(suggestedColRef); const suggestedCol = docModel.columns.getRowModel(suggestedColRef);
const suggestedTableId = suggestedCol.table().tableId(); suggestedTableId = suggestedCol.table().tableId();
if (optTableId && suggestedTableId !== optTableId) { if (optTableId && suggestedTableId !== optTableId) {
console.warn("Inappropriate column received from findColFromValues"); console.warn("Inappropriate column received from findColFromValues");
break; break;
}
} }
colInfo.type = `${toType}:${suggestedTableId}`; colInfo.type = `${toType}:${suggestedTableId}`;
colInfo.visibleCol = suggestedColRef; colInfo.visibleCol = suggestedColRef;
@ -163,11 +174,9 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
} }
} }
const newOptions = UserType.mergeOptions(widgetOptions || {}, colInfo.type);
if (widgetOptions) { if (widgetOptions) {
colInfo.widgetOptions = JSON.stringify(widgetOptions); colInfo.widgetOptions = JSON.stringify(widgetOptions);
} }
colInfo.formula = getDefaultFormula(docModel, origCol, colInfo.type, colInfo.visibleCol, newOptions);
return colInfo; return colInfo;
} }
@ -184,62 +193,6 @@ export async function setDisplayFormula(
} }
} }
// Given the original column and info about the new column, returns the formula to use for the
// transform column to do the transformation.
export function getDefaultFormula(
docModel: DocModel, origCol: ColumnRec, newType: string,
newVisibleCol: number, newWidgetOptions: any): string {
const colId = origCol.colId();
const oldVisibleColName = isReferenceCol(origCol) ?
getVisibleColName(docModel, origCol.visibleCol()) : undefined;
let origValFormula = oldVisibleColName ?
// The `str()` below converts AltText to plain text.
`($${colId}.${oldVisibleColName}
if ISREF($${colId}) or ISREFLIST($${colId})
else str($${colId}))`
: `$${colId}`;
if (origCol.type.peek() === 'ChoiceList') {
origValFormula = `grist.ChoiceList.toString($${colId})`;
}
const toTypePure: string = gristTypes.extractTypeFromColType(newType);
// The args are used to construct the call to grist.TYPE.typeConvert(value, [params]).
// Optional parameters depend on the type; see sandbox/grist/usertypes.py
const args: string[] = [origValFormula];
switch (toTypePure) {
case 'Ref':
case 'RefList':
{
const table = gutil.removePrefix(newType, toTypePure + ":");
args.push(table || 'None');
const visibleColName = getVisibleColName(docModel, newVisibleCol);
if (visibleColName) {
args.push(q(visibleColName));
}
break;
}
case 'Date': {
args.push(q(newWidgetOptions.dateFormat));
break;
}
case 'DateTime': {
const timezone = gutil.removePrefix(newType, "DateTime:") || '';
const format = newWidgetOptions.dateFormat + ' ' + newWidgetOptions.timeFormat;
args.push(q(format), q(timezone));
break;
}
}
return `grist.${gristTypes.getGristType(toTypePure)}.typeConvert(${args.join(', ')})`;
}
function q(value: string): string {
return "'" + value.replace(/'/g, "\\'") + "'";
}
// Returns the name of the visibleCol given its rowId. // Returns the name of the visibleCol given its rowId.
function getVisibleColName(docModel: DocModel, visibleColRef: number): string|undefined { function getVisibleColName(docModel: DocModel, visibleColRef: number): string|undefined {
return visibleColRef ? docModel.columns.getRowModel(visibleColRef).colId() : undefined; return visibleColRef ? docModel.columns.getRowModel(visibleColRef).colId() : undefined;

72
app/client/components/TypeTransform.ts
View File

@ -10,13 +10,12 @@ import {ColumnTransform} from 'app/client/components/ColumnTransform';
import {GristDoc} from 'app/client/components/GristDoc'; import {GristDoc} from 'app/client/components/GristDoc';
import * as TypeConversion from 'app/client/components/TypeConversion'; import * as TypeConversion from 'app/client/components/TypeConversion';
import {reportError} from 'app/client/models/errors'; import {reportError} from 'app/client/models/errors';
import * as modelUtil from 'app/client/models/modelUtil';
import {cssButtonRow} from 'app/client/ui/RightPanel'; import {cssButtonRow} from 'app/client/ui/RightPanel';
import {basicButton, primaryButton} from 'app/client/ui2018/buttons'; import {basicButton, primaryButton} from 'app/client/ui2018/buttons';
import {testId} from 'app/client/ui2018/cssVars'; import {testId} from 'app/client/ui2018/cssVars';
import {FieldBuilder} from 'app/client/widgets/FieldBuilder'; import {FieldBuilder} from 'app/client/widgets/FieldBuilder';
import {NewAbstractWidget} from 'app/client/widgets/NewAbstractWidget'; import {NewAbstractWidget} from 'app/client/widgets/NewAbstractWidget';
import {ColValues} from 'app/common/DocActions'; import {ColValues, UserAction} from 'app/common/DocActions';
import {Computed, dom, fromKo, Observable} from 'grainjs'; import {Computed, dom, fromKo, Observable} from 'grainjs';
import isEmpty = require('lodash/isEmpty'); import isEmpty = require('lodash/isEmpty');
import pickBy = require('lodash/pickBy'); import pickBy = require('lodash/pickBy');
@ -86,15 +85,6 @@ export class TypeTransform extends ColumnTransform {
); );
} }
protected async resetToDefaultFormula() {
if (!this.isFinalizing()) {
const toType = this.transformColumn.type.peek();
const formula = TypeConversion.getDefaultFormula(this.gristDoc.docModel, this.origColumn,
toType, this.field.visibleColRef(), this.field.widgetOptionsJson());
await modelUtil.setSaveValue(this.transformColumn.formula, formula);
}
}
/** /**
* Overrides parent method to initialize the transform column with guesses as to the particular * Overrides parent method to initialize the transform column with guesses as to the particular
* type and column options. * type and column options.
@ -103,20 +93,55 @@ export class TypeTransform extends ColumnTransform {
protected async addTransformColumn(toType: string) { protected async addTransformColumn(toType: string) {
const docModel = this.gristDoc.docModel; const docModel = this.gristDoc.docModel;
const colInfo = await TypeConversion.prepTransformColInfo(docModel, this.origColumn, this.origDisplayCol, toType); const colInfo = await TypeConversion.prepTransformColInfo(docModel, this.origColumn, this.origDisplayCol, toType);
const newColInfo = await this._tableData.sendTableAction(['AddColumn', 'gristHelper_Transform', colInfo]); const newColInfos = await this._tableData.sendTableActions([
const tcol = docModel.columns.getRowModel(newColInfo.colRef); ['AddColumn', 'gristHelper_Converted', {...colInfo, isFormula: false, formula: ''}],
await TypeConversion.setDisplayFormula(docModel, tcol); ['AddColumn', 'gristHelper_Transform', colInfo],
return newColInfo.colRef; ]);
const transformColRef = newColInfos[1].colRef;
this.transformColumn = docModel.columns.getRowModel(transformColRef);
await this.convertValues();
return transformColRef;
}
protected convertValuesActions(): UserAction[] {
const tableId = this._tableData.tableId;
const srcColId = this.origColumn.colId.peek();
const dstColId = "gristHelper_Converted";
const type = this.transformColumn.type.peek();
const widgetOptions = this.transformColumn.widgetOptions.peek();
const visibleColRef = this.transformColumn.visibleCol.peek();
return [['ConvertFromColumn', tableId, srcColId, dstColId, type, widgetOptions, visibleColRef]];
}
protected async convertValues() {
await Promise.all([
this.gristDoc.docData.sendActions(this.convertValuesActions()),
TypeConversion.setDisplayFormula(this.gristDoc.docModel, this.transformColumn),
]);
}
protected executeActions(): UserAction[] {
return [...this.convertValuesActions(), ...super.executeActions()];
} }
/** /**
* Overrides parent method to subscribe to changes to the transform column. * Overrides parent method to subscribe to changes to the transform column.
*/ */
protected postAddTransformColumn() { protected postAddTransformColumn() {
// When a user-initiated change is saved to type or widgetOptions, update the formula. // When a user-initiated change is saved to type or widgetOptions, reconvert the values
this.autoDispose(this.transformColumn.type.subscribe(this.resetToDefaultFormula, this, "save")); // Need to subscribe to both 'change' and 'save' for type which can come from setting the type itself
this.autoDispose(this.transformColumn.visibleCol.subscribe(this.resetToDefaultFormula, this, "save")); // or e.g. a change to DateTime timezone.
this.autoDispose(this.field.widgetOptionsJson.subscribe(this.resetToDefaultFormula, this, "save")); this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "change"));
this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "save"));
this.autoDispose(this.transformColumn.visibleCol.subscribe(this.convertValues, this, "save"));
this.autoDispose(this.field.widgetOptionsJson.subscribe(this.convertValues, this, "save"));
}
/**
* Overrides parent method to delete extra column
*/
protected cleanup() {
void this._tableData.sendTableAction(['RemoveColumn', 'gristHelper_Converted']);
} }
/** /**
@ -129,9 +154,10 @@ export class TypeTransform extends ColumnTransform {
const tcol = this.transformColumn; const tcol = this.transformColumn;
const changedInfo = pickBy(colInfo, (val, key) => const changedInfo = pickBy(colInfo, (val, key) =>
(val !== tcol[key as keyof TypeConversion.ColInfo].peek())); (val !== tcol[key as keyof TypeConversion.ColInfo].peek()));
return Promise.all([ if (!isEmpty(changedInfo)) {
isEmpty(changedInfo) ? undefined : tcol.updateColValues(changedInfo as ColValues), // Update the transform column, particularly the type.
TypeConversion.setDisplayFormula(docModel, tcol, changedInfo.visibleCol) // This will trigger the subscription in postAddTransformColumn and lead to calling convertValues.
]); await tcol.updateColValues(changedInfo as ColValues);
}
} }
} }

2
app/client/models/DocModel.ts
View File

@ -22,7 +22,7 @@ import {urlState} from 'app/client/models/gristUrlState';
import * as MetaRowModel from 'app/client/models/MetaRowModel'; import * as MetaRowModel from 'app/client/models/MetaRowModel';
import * as MetaTableModel from 'app/client/models/MetaTableModel'; import * as MetaTableModel from 'app/client/models/MetaTableModel';
import * as rowset from 'app/client/models/rowset'; import * as rowset from 'app/client/models/rowset';
import {isHiddenTable} from 'app/client/models/isHiddenTable'; import {isHiddenTable} from 'app/common/isHiddenTable';
import {schema, SchemaTypes} from 'app/common/schema'; import {schema, SchemaTypes} from 'app/common/schema';
import {ACLRuleRec, createACLRuleRec} from 'app/client/models/entities/ACLRuleRec'; import {ACLRuleRec, createACLRuleRec} from 'app/client/models/entities/ACLRuleRec';

52
app/client/models/entities/ColumnRec.ts
View File

@ -2,8 +2,13 @@ import {KoArray} from 'app/client/lib/koArray';
import {DocModel, IRowModel, recordSet, refRecord, TableRec, ViewFieldRec} from 'app/client/models/DocModel'; import {DocModel, IRowModel, recordSet, refRecord, TableRec, ViewFieldRec} from 'app/client/models/DocModel';
import {jsonObservable, ObjObservable} from 'app/client/models/modelUtil'; import {jsonObservable, ObjObservable} from 'app/client/models/modelUtil';
import * as gristTypes from 'app/common/gristTypes'; import * as gristTypes from 'app/common/gristTypes';
import {getReferencedTableId, isFullReferencingType} from 'app/common/gristTypes'; import {getReferencedTableId} from 'app/common/gristTypes';
import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter'; import {
BaseFormatter,
createFullFormatterRaw,
createVisibleColFormatterRaw,
FullFormatterArgs
} from 'app/common/ValueFormatter';
import * as ko from 'knockout'; import * as ko from 'knockout';
// Represents a column in a user-defined table. // Represents a column in a user-defined table.
@ -124,38 +129,23 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void {
// Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol // Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol
// associated with this column. If no visible column available, return formatting for the column itself. // associated with this column. If no visible column available, return formatting for the column itself.
this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this, docModel)); this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'vcol'));
this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, this.visibleColFormatter())); this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'full'));
}
export function visibleColFormatterForRec(
rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel
): BaseFormatter {
const vcol = rec.visibleColModel();
const documentSettings = docModel.docInfoRow.documentSettingsJson();
const type = colRec.type();
if (isFullReferencingType(type)) {
if (vcol.getRowId() === 0) {
// This column displays the Row ID, e.g. Table1[2]
// referencedTableId may actually be empty if the table is hidden
const referencedTableId: string = colRec.refTable()?.tableId() || "";
return createFormatter('Id', {tableId: referencedTableId}, documentSettings);
} else {
return createFormatter(vcol.type(), vcol.widgetOptionsJson(), documentSettings);
}
} else {
// For non-reference columns, there's no 'visible column' and we just return a regular formatter
return createFormatter(type, rec.widgetOptionsJson(), documentSettings);
}
} }
export function formatterForRec( export function formatterForRec(
rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, visibleColFormatter: BaseFormatter rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, kind: 'full' | 'vcol'
): BaseFormatter { ): BaseFormatter {
const type = colRec.type(); const vcol = rec.visibleColModel();
// Ref/RefList columns delegate most formatting to the visibleColFormatter const func = kind === 'full' ? createFullFormatterRaw : createVisibleColFormatterRaw;
const widgetOpts = {...rec.widgetOptionsJson(), visibleColFormatter}; const args: FullFormatterArgs = {
const documentSettings = docModel.docInfoRow.documentSettingsJson(); docData: docModel.docData,
return createFormatter(type, widgetOpts, documentSettings); type: colRec.type(),
widgetOpts: rec.widgetOptionsJson(),
visibleColType: vcol?.type(),
visibleColWidgetOpts: vcol?.widgetOptionsJson(),
docSettings: docModel.docInfoRow.documentSettingsJson(),
};
return func(args);
} }

9
app/client/models/entities/ViewFieldRec.ts
View File

@ -1,5 +1,5 @@
import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel'; import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel';
import {formatterForRec, visibleColFormatterForRec} from 'app/client/models/entities/ColumnRec'; import {formatterForRec} from 'app/client/models/entities/ColumnRec';
import * as modelUtil from 'app/client/models/modelUtil'; import * as modelUtil from 'app/client/models/modelUtil';
import * as UserType from 'app/client/widgets/UserType'; import * as UserType from 'app/client/widgets/UserType';
import {DocumentSettings} from 'app/common/DocumentSettings'; import {DocumentSettings} from 'app/common/DocumentSettings';
@ -172,13 +172,14 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void
// Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol // Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol
// associated with this field. If no visible column available, return formatting for the field itself. // associated with this field. If no visible column available, return formatting for the field itself.
this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this.column(), docModel)); this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'vcol'));
this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, this.visibleColFormatter())); this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'full'));
this.createValueParser = function() { this.createValueParser = function() {
const fieldRef = this.useColOptions.peek() ? undefined : this.id.peek(); const fieldRef = this.useColOptions.peek() ? undefined : this.id.peek();
return createParser(docModel.docData, this.colRef.peek(), fieldRef); const parser = createParser(docModel.docData, this.colRef.peek(), fieldRef);
return parser.cleanParse.bind(parser);
}; };
// The widgetOptions to read and write: either the column's or the field's own. // The widgetOptions to read and write: either the column's or the field's own.

2
app/client/ui/Pages.ts
View File

@ -3,7 +3,7 @@ import { duplicatePage } from "app/client/components/duplicatePage";
import { GristDoc } from "app/client/components/GristDoc"; import { GristDoc } from "app/client/components/GristDoc";
import { PageRec } from "app/client/models/DocModel"; import { PageRec } from "app/client/models/DocModel";
import { urlState } from "app/client/models/gristUrlState"; import { urlState } from "app/client/models/gristUrlState";
import { isHiddenTable } from 'app/client/models/isHiddenTable'; import { isHiddenTable } from 'app/common/isHiddenTable';
import * as MetaTableModel from "app/client/models/MetaTableModel"; import * as MetaTableModel from "app/client/models/MetaTableModel";
import { find as findInTree, fromTableData, TreeItemRecord, TreeRecord, import { find as findInTree, fromTableData, TreeItemRecord, TreeRecord,
TreeTableData} from "app/client/models/TreeModel"; TreeTableData} from "app/client/models/TreeModel";

255
app/common/ValueConverter.ts Normal file
View File

@ -0,0 +1,255 @@
import {DocData} from 'app/common/DocData';
import * as gristTypes from 'app/common/gristTypes';
import {isList} from 'app/common/gristTypes';
import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFormatter';
import {
createParserOrFormatterArgumentsRaw,
createParserRaw,
ReferenceListParser,
ReferenceParser,
ValueParser
} from 'app/common/ValueParser';
import {CellValue, GristObjCode} from 'app/plugin/GristData';
/**
* Base class for converting values from one type to another with the convert() method.
* Has a formatter for the source column
* and a parser for the destination column.
*
* The default convert() is for non-list destination types, so if the source value
* is a list it only converts nicely if the list contains exactly one element.
*/
export class ValueConverter {
constructor(public formatter: BaseFormatter, public parser: ValueParser) {
}
public convert(value: any): any {
if (isList(value)) {
if (value.length === 1) {
// Empty list: ['L']
return null;
} else if (value.length === 2) {
// Singleton list: ['L', value]
// Convert just that one value.
value = value[1];
} else {
// List with multiple values. Since we're converting to just one value,
// format the whole thing as text, which is an error for most types.
return this.formatter.formatAny(value);
}
}
return this.convertInner(value);
}
protected convertInner(value: any): any {
const formatted = this.formatter.formatAny(value);
return this.parser.cleanParse(formatted);
}
}
/**
* Base class for converting to a list type (Reference List or Choice List).
*
* Wraps single values in a list, and converts lists elementwise.
*/
class ListConverter extends ValueConverter {
// Don't parse strings like "Smith, John" which may look like lists but represent a single choice.
// TODO this works when the source is a Choice column, but not when it's a Reference to a Choice column.
// But the guessed choices are also broken in that case.
private _choices: Set<string> = new Set((this.formatter.widgetOpts as any).choices || []);
public convert(value: any): any {
if (typeof value === "string" && !this._choices.has(value)) {
// Parse CSV/JSON
return this.parser.cleanParse(value);
}
const values = isList(value) ? value.slice(1) : [value];
if (!values.length || value == null) {
return null;
}
return this.handleValues(value, values.map(v => this.convertInner(v)));
}
protected handleValues(originalValue: any, values: any[]) {
return ['L', ...values];
}
}
class ChoiceListConverter extends ListConverter {
/**
* Convert each source value to a 'Choice'
*/
protected convertInner(value: any): any {
return this.formatter.formatAny(value);
}
}
class ReferenceListConverter extends ListConverter {
private _innerConverter = new ReferenceConverter(
this.formatter,
new ReferenceParser("Ref", this.parser.widgetOpts, this.parser.docSettings),
);
constructor(public formatter: BaseFormatter, public parser: ReferenceListParser) {
super(formatter, parser);
// Prevent the parser from looking up reference values in the frontend.
// Leave it to the data engine which has a much more efficient algorithm for long lists of values.
delete parser.tableData;
}
public handleValues(originalValue: any, values: any[]): any {
const result = [];
let lookupColumn: string = "";
const raw = this.formatter.formatAny(originalValue); // AltText if the reference lookup fails
for (const value of values) {
if (typeof value === "string") {
// Failed to parse one of the references, so return a raw string for the whole thing
return raw;
} else {
// value is a lookup tuple: ['l', value, options]
result.push(value[1]);
lookupColumn = value[2].column;
}
}
return ['l', result, {column: lookupColumn, raw}];
}
/**
* Convert each source value to a 'Reference'
*/
protected convertInner(value: any): any {
return this._innerConverter.convert(value);
}
}
class ReferenceConverter extends ValueConverter {
private _innerConverter: ValueConverter = createConverter(this.formatter, this.parser.visibleColParser);
constructor(public formatter: BaseFormatter, public parser: ReferenceParser) {
super(formatter, parser);
// Prevent the parser from looking up reference values in the frontend.
// Leave it to the data engine which has a much more efficient algorithm for long lists of values.
delete parser.tableData;
}
protected convertInner(value: any): any {
// Convert to the type of the visible column.
const converted = this._innerConverter.convert(value);
return this.parser.lookup(converted, this.formatter.formatAny(value));
}
}
class NumericConverter extends ValueConverter {
protected convertInner(value: any): any {
if (typeof value === "boolean") {
return value ? 1 : 0;
}
return super.convertInner(value);
}
}
class DateConverter extends ValueConverter {
private _sourceType = gristTypes.extractInfoFromColType(this.formatter.type);
protected convertInner(value: any): any {
// When converting Date->DateTime, DateTime->Date, or between DateTime timezones,
// it's important to send an encoded Date/DateTime object rather than just a timestamp number
// so that the data engine knows what to do in do_convert, especially regarding timezones.
// If the source column is a Reference to a Date/DateTime then `value` is already
// an encoded object from the display column which has type Any.
value = gristTypes.reencodeAsAny(value, this._sourceType);
if (Array.isArray(value) && (
value[0] === GristObjCode.Date ||
value[0] === GristObjCode.DateTime
)) {
return value;
}
return super.convertInner(value);
}
}
export const valueConverterClasses: { [type: string]: typeof ValueConverter } = {
Date: DateConverter,
DateTime: DateConverter,
ChoiceList: ChoiceListConverter,
Ref: ReferenceConverter,
RefList: ReferenceListConverter,
Numeric: NumericConverter,
Int: NumericConverter,
};
export function createConverter(formatter: BaseFormatter, parser: ValueParser) {
const cls = valueConverterClasses[gristTypes.extractTypeFromColType(parser.type)] || ValueConverter;
return new cls(formatter, parser);
}
/**
* Used by the ConvertFromColumn user action in the data engine.
* The higher order function separates docData (passed by ActiveDoc)
* from the arguments passed to call_external in Python.
*/
export function convertFromColumn(docData: DocData) {
return function(
sourceColRef: number,
type: string,
widgetOpts: string,
visibleColRef: number,
values: ReadonlyArray<CellValue>,
displayColValues?: ReadonlyArray<CellValue>,
): CellValue[] {
const formatter = createFullFormatterFromDocData(docData, sourceColRef);
const parser = createParserRaw(
...createParserOrFormatterArgumentsRaw(docData, type, widgetOpts, visibleColRef)
);
const converter = createConverter(formatter, parser);
return convertValues(converter, values, displayColValues || values);
};
}
export function convertValues(
converter: ValueConverter,
// Raw values from the actual column, e.g. row IDs for reference columns
values: ReadonlyArray<CellValue>,
// Values from the display column, which is the same as the raw values for non-referencing columns.
// In almost all cases these are the values that actually matter and get converted.
displayColValues: ReadonlyArray<CellValue>,
): CellValue[] {
// Converting Ref <-> RefList without changing the target table is a special case - see prepTransformColInfo.
// In this case we deal with the actual row IDs stored in the real column,
// whereas in all other cases we use display column values.
const sourceType = gristTypes.extractInfoFromColType(converter.formatter.type);
const targetType = gristTypes.extractInfoFromColType(converter.parser.type);
const refToRefList = (
sourceType.type === "Ref" &&
targetType.type === "RefList" &&
sourceType.tableId === targetType.tableId
);
const refListToRef = (
sourceType.type === "RefList" &&
targetType.type === "Ref" &&
sourceType.tableId === targetType.tableId
);
return displayColValues.map((displayVal, i) => {
const actualValue = values[i];
if (refToRefList && typeof actualValue === "number") {
if (actualValue === 0) {
return null;
} else {
return ["L", actualValue];
}
} else if (refListToRef && isList(actualValue)) {
if (actualValue.length === 1) {
// Empty list: ['L']
return 0;
} else if (actualValue.length === 2) {
// Singleton list: ['L', rowId]
return actualValue[1];
}
}
return converter.convert(displayVal);
});
}

72
app/common/ValueFormatter.ts
View File

@ -2,11 +2,14 @@
import {csvEncodeRow} from 'app/common/csvFormat'; import {csvEncodeRow} from 'app/common/csvFormat';
import {CellValue} from 'app/common/DocActions'; import {CellValue} from 'app/common/DocActions';
import {DocData} from 'app/common/DocData';
import {DocumentSettings} from 'app/common/DocumentSettings'; import {DocumentSettings} from 'app/common/DocumentSettings';
import {getReferencedTableId, isList} from 'app/common/gristTypes';
import * as gristTypes from 'app/common/gristTypes'; import * as gristTypes from 'app/common/gristTypes';
import {getReferencedTableId, isList} from 'app/common/gristTypes';
import * as gutil from 'app/common/gutil'; import * as gutil from 'app/common/gutil';
import {isHiddenTable} from 'app/common/isHiddenTable';
import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat'; import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
import {createParserOrFormatterArguments, ReferenceParsingOptions} from 'app/common/ValueParser';
import {GristObjCode} from 'app/plugin/GristData'; import {GristObjCode} from 'app/plugin/GristData';
import {decodeObject, GristDateTime} from 'app/plugin/objtypes'; import {decodeObject, GristDateTime} from 'app/plugin/objtypes';
import * as moment from 'moment-timezone'; import * as moment from 'moment-timezone';
@ -280,3 +283,70 @@ export function createFormatter(type: string, widgetOpts: FormatOptions, docSett
const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter; const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter;
return new ctor(type, widgetOpts, docSettings); return new ctor(type, widgetOpts, docSettings);
} }
export interface FullFormatterArgs {
docData: DocData;
type: string;
widgetOpts: FormatOptions;
visibleColType: string;
visibleColWidgetOpts: FormatOptions;
docSettings: DocumentSettings;
}
/**
* Returns a constructor
* with a format function that can properly convert a value passed to it into the
* right format for that column.
*
* Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
* instead of the table column.
*/
export function createFullFormatterFromDocData(
docData: DocData,
colRef: number,
fieldRef?: number,
): BaseFormatter {
const [type, widgetOpts, docSettings] = createParserOrFormatterArguments(docData, colRef, fieldRef);
const {visibleColType, visibleColWidgetOpts} = widgetOpts as ReferenceParsingOptions;
return createFullFormatterRaw({
docData,
type,
widgetOpts,
visibleColType,
visibleColWidgetOpts,
docSettings,
});
}
export function createFullFormatterRaw(args: FullFormatterArgs) {
const {type, widgetOpts, docSettings} = args;
const visibleColFormatter = createVisibleColFormatterRaw(args);
return createFormatter(type, {...widgetOpts, visibleColFormatter}, docSettings);
}
export function createVisibleColFormatterRaw(
{
docData,
docSettings,
type,
visibleColType,
visibleColWidgetOpts,
widgetOpts
}: FullFormatterArgs
): BaseFormatter {
let referencedTableId = gristTypes.getReferencedTableId(type);
if (!referencedTableId) {
return createFormatter(type, widgetOpts, docSettings);
} else if (visibleColType) {
return createFormatter(visibleColType, visibleColWidgetOpts, docSettings);
} else {
// This column displays the Row ID, e.g. Table1[2]
// Make referencedTableId empty if the table is hidden
const tablesData = docData.getMetaTable("_grist_Tables");
const tableRef = tablesData.findRow("tableId", referencedTableId);
if (isHiddenTable(tablesData, tableRef)) {
referencedTableId = "";
}
return createFormatter('Id', {tableId: referencedTableId}, docSettings);
}
}

88
app/common/ValueParser.ts
View File

@ -33,6 +33,9 @@ export class ValueParser {
} }
class IdentityParser extends ValueParser {
}
/** /**
* Same as basic Value parser, but will return null if a value is an empty string. * Same as basic Value parser, but will return null if a value is an empty string.
*/ */
@ -117,7 +120,7 @@ class ChoiceListParser extends ValueParser {
* stored on the field. These have to be specially derived * stored on the field. These have to be specially derived
* for referencing columns. See createParser. * for referencing columns. See createParser.
*/ */
interface ReferenceParsingOptions { export interface ReferenceParsingOptions {
visibleColId: string; visibleColId: string;
visibleColType: string; visibleColType: string;
visibleColWidgetOpts: FormatOptions; visibleColWidgetOpts: FormatOptions;
@ -129,18 +132,22 @@ interface ReferenceParsingOptions {
export class ReferenceParser extends ValueParser { export class ReferenceParser extends ValueParser {
public widgetOpts: ReferenceParsingOptions; public widgetOpts: ReferenceParsingOptions;
public tableData = this.widgetOpts.tableData;
protected _visibleColId = this.widgetOpts.visibleColId; public visibleColParser = createParserRaw(
protected _tableData = this.widgetOpts.tableData;
protected _visibleColParser = createParserRaw(
this.widgetOpts.visibleColType, this.widgetOpts.visibleColType,
this.widgetOpts.visibleColWidgetOpts, this.widgetOpts.visibleColWidgetOpts,
this.docSettings, this.docSettings,
); );
protected _visibleColId = this.widgetOpts.visibleColId;
public parse(raw: string): any { public parse(raw: string): any {
let value = this._visibleColParser(raw); const value = this.visibleColParser.cleanParse(raw);
if (!value || !raw) { return this.lookup(value, raw);
}
public lookup(value: any, raw: string): any {
if (value == null || value === "" || !raw) {
return 0; // default value for a reference column return 0; // default value for a reference column
} }
@ -154,7 +161,7 @@ export class ReferenceParser extends ValueParser {
} }
} }
if (!this._tableData?.isLoaded) { if (!this.tableData?.isLoaded) {
const options: { column: string, raw?: string } = {column: this._visibleColId}; const options: { column: string, raw?: string } = {column: this._visibleColId};
if (value !== raw) { if (value !== raw) {
options.raw = raw; options.raw = raw;
@ -162,7 +169,7 @@ export class ReferenceParser extends ValueParser {
return ['l', value, options]; return ['l', value, options];
} }
return this._tableData.findMatchingRowId({[this._visibleColId]: value}) || raw; return this.tableData.findMatchingRowId({[this._visibleColId]: value}) || raw;
} }
} }
@ -178,7 +185,7 @@ export class ReferenceListParser extends ReferenceParser {
// csvDecodeRow should never raise an exception // csvDecodeRow should never raise an exception
values = csvDecodeRow(raw); values = csvDecodeRow(raw);
} }
values = values.map(v => typeof v === "string" ? this._visibleColParser(v) : encodeObject(v)); values = values.map(v => typeof v === "string" ? this.visibleColParser.cleanParse(v) : encodeObject(v));
if (!values.length || !raw) { if (!values.length || !raw) {
return null; // null is the default value for a reference list column return null; // null is the default value for a reference list column
@ -194,7 +201,7 @@ export class ReferenceListParser extends ReferenceParser {
} }
} }
if (!this._tableData?.isLoaded) { if (!this.tableData?.isLoaded) {
const options: { column: string, raw?: string } = {column: this._visibleColId}; const options: { column: string, raw?: string } = {column: this._visibleColId};
if (!(values.length === 1 && values[0] === raw)) { if (!(values.length === 1 && values[0] === raw)) {
options.raw = raw; options.raw = raw;
@ -204,7 +211,7 @@ export class ReferenceListParser extends ReferenceParser {
const rowIds: number[] = []; const rowIds: number[] = [];
for (const value of values) { for (const value of values) {
const rowId = this._tableData.findMatchingRowId({[this._visibleColId]: value}); const rowId = this.tableData.findMatchingRowId({[this._visibleColId]: value});
if (rowId) { if (rowId) {
rowIds.push(rowId); rowIds.push(rowId);
} else { } else {
@ -228,27 +235,21 @@ export const valueParserClasses: { [type: string]: typeof ValueParser } = {
RefList: ReferenceListParser, RefList: ReferenceListParser,
}; };
const identity = (value: string) => value;
/** /**
* Returns a function which can parse strings into values appropriate for * Returns a ValueParser which can parse strings into values appropriate for
* a specific widget field or table column. * a specific widget field or table column.
* widgetOpts is usually the field/column's widgetOptions JSON * widgetOpts is usually the field/column's widgetOptions JSON
* but referencing columns need more than that, see ReferenceParsingOptions above. * but referencing columns need more than that, see ReferenceParsingOptions above.
*/ */
export function createParserRaw( export function createParserRaw(
type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings
): (value: string) => any { ): ValueParser {
const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)]; const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)] || IdentityParser;
if (cls) { return new cls(type, widgetOpts, docSettings);
const parser = new cls(type, widgetOpts, docSettings);
return parser.cleanParse.bind(parser);
}
return identity;
} }
/** /**
* Returns a function which can parse strings into values appropriate for * Returns a ValueParser which can parse strings into values appropriate for
* a specific widget field or table column. * a specific widget field or table column.
* *
* Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
@ -258,23 +259,46 @@ export function createParser(
docData: DocData, docData: DocData,
colRef: number, colRef: number,
fieldRef?: number, fieldRef?: number,
): (value: string) => any { ): ValueParser {
return createParserRaw(...createParserOrFormatterArguments(docData, colRef, fieldRef));
}
/**
* Returns arguments suitable for createParserRaw or createFormatter. Only for internal use.
*
* Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
* instead of the table column.
*/
export function createParserOrFormatterArguments(
docData: DocData,
colRef: number,
fieldRef?: number,
): [string, object, DocumentSettings] {
const columnsTable = docData.getMetaTable('_grist_Tables_column'); const columnsTable = docData.getMetaTable('_grist_Tables_column');
const fieldsTable = docData.getMetaTable('_grist_Views_section_field'); const fieldsTable = docData.getMetaTable('_grist_Views_section_field');
const docInfoTable = docData.getMetaTable('_grist_DocInfo');
const col = columnsTable.getRecord(colRef)!; const col = columnsTable.getRecord(colRef)!;
let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col; let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col;
if (fieldRef) { if (fieldRef) {
fieldOrCol = fieldsTable.getRecord(fieldRef) || col; fieldOrCol = fieldsTable.getRecord(fieldRef) || col;
} }
const widgetOpts = safeJsonParse(fieldOrCol.widgetOptions, {}); return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol);
}
export function createParserOrFormatterArgumentsRaw(
docData: DocData,
type: string,
widgetOptions: string,
visibleColRef: number,
): [string, object, DocumentSettings] {
const columnsTable = docData.getMetaTable('_grist_Tables_column');
const docInfoTable = docData.getMetaTable('_grist_DocInfo');
const widgetOpts = safeJsonParse(widgetOptions, {});
const type = col.type;
if (isFullReferencingType(type)) { if (isFullReferencingType(type)) {
const vcol = columnsTable.getRecord(fieldOrCol.visibleCol); const vcol = columnsTable.getRecord(visibleColRef);
widgetOpts.visibleColId = vcol?.colId || 'id'; widgetOpts.visibleColId = vcol?.colId || 'id';
widgetOpts.visibleColType = vcol?.type; widgetOpts.visibleColType = vcol?.type;
widgetOpts.visibleColWidgetOpts = safeJsonParse(vcol?.widgetOptions || '', {}); widgetOpts.visibleColWidgetOpts = safeJsonParse(vcol?.widgetOptions || '', {});
@ -284,7 +308,7 @@ export function createParser(
const docInfo = docInfoTable.getRecord(1); const docInfo = docInfoTable.getRecord(1);
const docSettings = safeJsonParse(docInfo!.documentSettings, {}) as DocumentSettings; const docSettings = safeJsonParse(docInfo!.documentSettings, {}) as DocumentSettings;
return createParserRaw(type, widgetOpts, docSettings); return [type, widgetOpts, docSettings];
} }
/** /**
@ -311,12 +335,12 @@ function parseColValues<T extends ColValues | BulkColValues>(
const parser = createParser(docData, colRef); const parser = createParser(docData, colRef);
// Optimisation: If there's no special parser for this column type, do nothing // Optimisation: If there's no special parser for this column type, do nothing
if (parser === identity) { if (parser instanceof IdentityParser) {
return values; return values;
} }
function parseIfString(val: any) { function parseIfString(val: any) {
return typeof val === "string" ? parser(val) : val; return typeof val === "string" ? parser.cleanParse(val) : val;
} }
if (bulk) { if (bulk) {

6
app/client/models/isHiddenTable.ts → app/common/isHiddenTable.ts
View File

@ -1,11 +1,11 @@
import {RowId} from 'app/client/models/rowset'; import {UIRowId} from 'app/common/UIRowId';
import {TableData} from 'app/client/models/TableData'; import {TableData} from "./TableData";
/** /**
* Return whether a table identified by the rowId of its metadata record, should normally be * Return whether a table identified by the rowId of its metadata record, should normally be
* hidden from the user (e.g. as an option in the page-widget picker). * hidden from the user (e.g. as an option in the page-widget picker).
*/ */
export function isHiddenTable(tablesData: TableData, tableRef: RowId): boolean { export function isHiddenTable(tablesData: TableData, tableRef: UIRowId): boolean {
const tableId = tablesData.getValue(tableRef, 'tableId') as string|undefined; const tableId = tablesData.getValue(tableRef, 'tableId') as string|undefined;
return tablesData.getValue(tableRef, 'summarySourceTable') !== 0 || return tablesData.getValue(tableRef, 'summarySourceTable') !== 0 ||
Boolean(tableId?.startsWith('GristHidden')); Boolean(tableId?.startsWith('GristHidden'));

7
app/server/lib/ActiveDoc.ts
View File

@ -41,6 +41,7 @@ import {schema, SCHEMA_VERSION} from 'app/common/schema';
import {MetaRowRecord} from 'app/common/TableData'; import {MetaRowRecord} from 'app/common/TableData';
import {FetchUrlOptions, UploadResult} from 'app/common/uploads'; import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI'; import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
import {convertFromColumn} from 'app/common/ValueConverter';
import {parseUserAction} from 'app/common/ValueParser'; import {parseUserAction} from 'app/common/ValueParser';
import {ParseOptions} from 'app/plugin/FileParserAPI'; import {ParseOptions} from 'app/plugin/FileParserAPI';
import {GristDocAPI} from 'app/plugin/GristAPI'; import {GristDocAPI} from 'app/plugin/GristAPI';
@ -1717,6 +1718,12 @@ export class ActiveDoc extends EventEmitter {
logTimes: true, logTimes: true,
logMeta: {docId: this._docName}, logMeta: {docId: this._docName},
preferredPythonVersion, preferredPythonVersion,
sandboxOptions: {
exports: {
convertFromColumn: (...args: Parameters<ReturnType<typeof convertFromColumn>>) =>
convertFromColumn(this.docData!)(...args)
}
},
}); });
} }
} }

1
app/server/lib/GranularAccess.ts
View File

@ -87,6 +87,7 @@ const SPECIAL_ACTIONS = new Set(['InitNewDoc',
'TransformAndFinishImport', 'TransformAndFinishImport',
'AddView', 'AddView',
'CopyFromColumn', 'CopyFromColumn',
'ConvertFromColumn',
'AddHiddenColumn', 'AddHiddenColumn',
]); ]);

3
app/server/lib/ISandbox.ts
View File

@ -1,4 +1,5 @@
import * as log from 'app/server/lib/log'; import * as log from 'app/server/lib/log';
import {ISandboxOptions} from 'app/server/lib/NSandbox';
/** /**
* Starting to whittle down the options used when creating a sandbox, to leave more * Starting to whittle down the options used when creating a sandbox, to leave more
@ -17,6 +18,8 @@ export interface ISandboxCreationOptions {
importMount?: string; // if defined, make this path available read-only as "/importdir" importMount?: string; // if defined, make this path available read-only as "/importdir"
preferredPythonVersion?: '2' | '3'; preferredPythonVersion?: '2' | '3';
sandboxOptions?: Partial<ISandboxOptions>;
} }
export interface ISandbox { export interface ISandbox {

3
app/server/lib/NSandbox.ts
View File

@ -33,7 +33,7 @@ type SandboxMethod = (...args: any[]) => any;
* started by setting `useGristEntrypoint` (the only exception is * started by setting `useGristEntrypoint` (the only exception is
* in tests) which runs grist/main.py. * in tests) which runs grist/main.py.
*/ */
interface ISandboxOptions { export interface ISandboxOptions {
command?: string; // External program or container to call to run the sandbox. command?: string; // External program or container to call to run the sandbox.
args: string[]; // The arguments to pass to the python process. args: string[]; // The arguments to pass to the python process.
@ -404,6 +404,7 @@ export class NSandboxCreator implements ISandboxCreator {
preferredPythonVersion: this._preferredPythonVersion || options.preferredPythonVersion, preferredPythonVersion: this._preferredPythonVersion || options.preferredPythonVersion,
useGristEntrypoint: true, useGristEntrypoint: true,
importDir: options.importMount, importDir: options.importMount,
...options.sandboxOptions,
}; };
return new NSandbox(translatedOptions, spawners[this._flavor]); return new NSandbox(translatedOptions, spawners[this._flavor]);
} }

19
sandbox/grist/column.py
View File

@ -473,8 +473,23 @@ class BaseReferenceColumn(BaseColumn):
.get_column_rec(self.table_id, self.col_id).visibleCol.colId .get_column_rec(self.table_id, self.col_id).visibleCol.colId
or "id" or "id"
) )
target_value = self._target_table.get_column(col_id)._convert_raw_value(value) column = self._target_table.get_column(col_id)
return self._target_table.lookup_one_record(**{col_id: target_value}) # `value` is an object encoded for transmission from JS to Python,
# which is decoded to `decoded_value`.
# `raw_value` is the kind of value that would be stored in `column`.
# `rich_value` is the type of value used in formulas, especially with `lookupRecords`.
# For example, for a Date column, `raw_value` is a numerical timestamp
# and `rich_value` is a `datetime.date` object,
# assuming `value` isn't of an invalid type.
# However `value` could either be just a number
# (in which case `decoded_value` would be a number as well)
# or an encoded date (or even datetime) object like ['d', number]
# (in which case `decoded_value` would be a `datetime.date` object,
# which would get converted back to a number and then back to a date object again!)
decoded_value = objtypes.decode_object(value)
raw_value = column.convert(decoded_value)
rich_value = column._convert_raw_value(raw_value)
return self._target_table.lookup_one_record(**{col_id: rich_value})
class ReferenceColumn(BaseReferenceColumn): class ReferenceColumn(BaseReferenceColumn):

8
sandbox/grist/functions/info.py
View File

@ -500,6 +500,14 @@ def N(value):
return 0 return 0
def CURRENT_CONVERSION(rec):
"""
Special function used only when changing the type of a column.
Doesn't work in normal formulas.
"""
return rec.gristHelper_Converted
def NA(): def NA():
""" """
Returns the "value not available" error, `#N/A`. Returns the "value not available" error, `#N/A`.

6
sandbox/grist/functions/schedule.py
View File

@ -1,12 +1,16 @@
from datetime import datetime, timedelta from datetime import datetime, timedelta
import re import re
from .date import DATEADD, NOW, DTIME from .date import DATEADD, NOW, DTIME
from moment_parse import MONTH_NAMES, DAY_NAMES
# Limit exports to schedule, so that upper-case constants like MONTH_NAMES, DAY_NAMES don't end up # Limit exports to schedule, so that upper-case constants like MONTH_NAMES, DAY_NAMES don't end up
# exposed as if Excel-style functions (or break docs generation). # exposed as if Excel-style functions (or break docs generation).
__all__ = ['SCHEDULE'] __all__ = ['SCHEDULE']
MONTH_NAMES = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august',
'september', 'october', 'november', 'december']
# Regex list of lowercase weekdays with characters after the first three made optional
DAY_NAMES = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday']
def SCHEDULE(schedule, start=None, count=10, end=None): def SCHEDULE(schedule, start=None, count=10, end=None):
""" """
Returns the list of `datetime` objects generated according to the `schedule` string. Starts at Returns the list of `datetime` objects generated according to the `schedule` string. Starts at

6
sandbox/grist/moment.py
View File

@ -4,7 +4,6 @@ import marshal
from time import time from time import time
import bisect import bisect
import os import os
import moment_parse
import iso8601 import iso8601
import six import six
from six.moves import zip from six.moves import zip
@ -13,7 +12,6 @@ from six.moves import zip
ZoneRecord = namedtuple("ZoneRecord", ("name", "abbrs", "offsets", "untils")) ZoneRecord = namedtuple("ZoneRecord", ("name", "abbrs", "offsets", "untils"))
# moment.py mirrors core functionality of moment-timezone.js # moment.py mirrors core functionality of moment-timezone.js
# moment.py includes function parse, located and documented in moment_parse.py
# Documentation: http://momentjs.com/timezone/docs/ # Documentation: http://momentjs.com/timezone/docs/
EPOCH = datetime(1970, 1, 1) EPOCH = datetime(1970, 1, 1)
@ -67,10 +65,6 @@ def date_to_ts(date, timezone=None):
ts = (date - DATE_EPOCH).total_seconds() ts = (date - DATE_EPOCH).total_seconds()
return ts if not timezone else ts - timezone.offset(ts * 1000).total_seconds() return ts if not timezone else ts - timezone.offset(ts * 1000).total_seconds()
# Calls parse from moment_parse.py
def parse(date_string, parse_format, zonelabel='UTC'):
return moment_parse.parse(date_string, parse_format, zonelabel)
# Parses a datetime in the ISO format, YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM. Most parts are optional; # Parses a datetime in the ISO format, YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM. Most parts are optional;
# see https://pypi.org/project/iso8601/ for details. Returns a timestamp in seconds. # see https://pypi.org/project/iso8601/ for details. Returns a timestamp in seconds.
def parse_iso(date_string, timezone=None): def parse_iso(date_string, timezone=None):

161
sandbox/grist/moment_parse.py
View File

@ -1,161 +0,0 @@
import re
from collections import OrderedDict
from datetime import datetime
import moment
# Regex list of lowercase months with characters after the first three made optional
MONTH_NAMES = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august',
'september', 'october', 'november', 'december']
MONTHS = [m[:3]+"(?:"+m[3:]+")?" if len(m) > 3 else m[:3] for m in MONTH_NAMES]
# Regex list of lowercase weekdays with characters after the first three made optional
DAY_NAMES = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday']
WEEKDAYS = [d[:3]+"(?:"+d[3:]+")?" for d in DAY_NAMES]
# Acceptable format tokens mapped to what they should match in the date string
# Ordered so that larger configurations are matched first
DATE_TOKENS = OrderedDict([
("HH", r"(?P<H>\d{1,2})"), # 24 hr
("H", r"(?P<H>\d{1,2})"),
("hh", r"(?P<h>\d{1,2})"), # 12 hr
("h", r"(?P<h>\d{1,2})"),
("mm", r"(?P<m>\d{1,2})"), # min
("m", r"(?P<m>\d{1,2})"),
("A", r"(?P<A>[ap]m?)"), # am/pm
("a", r"(?P<A>[ap]m?)"),
("ss", r"(?P<s>\d{1,2})"), # sec
("s", r"(?P<s>\d{1,2})"),
("SSSSSS", r"(?P<S>\d{1,6})"), # fractional second
("SSSSS", r"(?P<S>\d{1,6})"),
("SSSS", r"(?P<S>\d{1,6})"),
("SSS", r"(?P<S>\d{1,6})"),
("SS", r"(?P<S>\d{1,6})"),
("S", r"(?P<S>\d{1,6})"),
("YYYY", r"(?P<YY>\d{4}|\d{2})"), # 4 or 2 digit year
("YY", r"(?P<YY>\d{2})"), # 2 digit year
("MMMM", r"(?P<MMM>" + ("|".join(MONTHS)) + ")"), # month name, abbr or not
("MMM", r"(?P<MMM>" + ("|".join(MONTHS)) + ")"),
("MM", r"(?P<M>\d{1,2})"), # month num
("M", r"(?P<M>\d{1,2})"),
("DD", r"(?P<D>\d{1,2})"), # day num
("Do", r"(?P<D>\d{1,2})(st|nd|rd|th)"),
("D", r"(?P<D>\d{1,2})"),
("dddd", r"(" + ("|".join(WEEKDAYS)) + ")"), # day name, abbr or not (ignored)
("ddd", r"(" + ("|".join(WEEKDAYS)) + ")")
])
DATE_TOKENS_REGEX = re.compile("("+("|".join(DATE_TOKENS))+")")
# List of separators to replace and match any standard date/time separators
SEP = r"[\s/.\-:,]*"
SEP_REGEX = re.compile(SEP)
SEP_REPLACEMENT = SEP.replace("\\", "\\\\")
# Maps date parse format to compile regex
FORMAT_CACHE = {}
# Parses date_string using parse_format in the style of moment.js
# See: http://momentjs.com/docs/#/parsing
# Supports the following tokens:
# H HH 0..23 24 hour time
# h hh 1..12 12 hour time used with a A.
# a A am pm Post or ante meridiem
# m mm 0..59 Minutes
# s ss 0..59 Seconds
# S SS SSS 0..999 Fractional seconds
# YYYY 2014 4 or 2 digit year
# YY 14 2 digit year
# M MM 1..12 Month number
# MMM MMMM Jan..December Month name in locale set by moment.locale()
# D DD 1..31 Day of month
# Do 1st..31st Day of month with ordinal
def parse(date_string, parse_format, zonelabel='UTC', override_current_date=None):
"""Parse a date string via a moment.js style parse format and a timezone string.
Supported tokens are documented above. Returns seconds since epoch"""
if parse_format in FORMAT_CACHE:
# Check if parse_format has been cache, and retrieve if so
parser = FORMAT_CACHE[parse_format]
else:
# e.g. "MM-YY" -> "(?P<mm>\d{1,2})-(?P<yy>\d{2})"
# Note that DATE_TOKENS is ordered so that the longer letter chains are recognized first
tokens = DATE_TOKENS_REGEX.split(parse_format)
tokens = [DATE_TOKENS[t] if t in DATE_TOKENS else SEP_REGEX.sub(SEP_REPLACEMENT, t)
for t in tokens]
# Compile new token string ignoring case (for month names)
parser = re.compile(''.join(tokens), re.I)
FORMAT_CACHE[parse_format] = parser
match = parser.match(date_string)
# Throw error if matching failed
if match is None:
raise Exception("Failed to parse %s with %s" % (date_string, parse_format))
# Create datetime from the results of parsing
current_date = override_current_date or moment.CURRENT_DATE
m = match.groupdict()
dt = datetime(
year=getYear(m, current_date.year),
month=getMonth(m, current_date.month),
day=int(m['D']) if ('D' in m) else current_date.day,
hour=getHour(m),
minute=int(m['m']) if ('m' in m) else 0,
second=int(m['s']) if ('s' in m) else 0,
microsecond=getMicrosecond(m)
)
# Parses the datetime with the given timezone to return the seconds since EPOCH
return moment.tz(dt, zonelabel).timestamp_s()
def getYear(match_dict, current_year):
if 'YYYY' in match_dict:
return int(match_dict['YYYY'])
elif 'YY' in match_dict:
match = match_dict['YY']
if len(match) == 2:
# Must guess on the century, choose so the result is closest to the current year
# The first year that could be meant by YY is the current year - 50.
first = current_year - 50
# We are seeking k such that 100k + YY is between first and first + 100.
# first <= 100k + YY < first + 100
# 0 <= 100k + YY - first < 100
# The value inside the comparison operators is precisely (YY - first) % 100.
# So we can calculate the century 100k as (YY - first) % 100 - (YY - first).
return first + (int(match) - first) % 100
else:
return int(match)
else:
return current_year
def getMonth(match_dict, current_month):
if 'M' in match_dict:
return int(match_dict['M'])
elif 'MMM' in match_dict:
return lazy_index(MONTHS, match_dict['MMM'][:3].lower()) + 1
else:
return current_month
def getHour(match_dict):
if 'H' in match_dict:
return int(match_dict['H'])
elif 'h' in match_dict:
hr = int(match_dict['h']) % 12
merid = 12 if 'A' in match_dict and match_dict['A'][0] == "p" else 0
return hr + merid
else:
return 0
def getMicrosecond(match_dict):
if 'S' in match_dict:
match = match_dict['S']
return int(match + ("0"*(6-len(match))) if len(match) < 6 else match[:6])
else:
return 0
# Gets the index of the first string from iter that starts with startswith
def lazy_index(l, startswith, missing=None):
for i, token in enumerate(l):
if token[:len(startswith)] == startswith:
return i
return missing

79
sandbox/grist/test_moment.py
View File

@ -1,7 +1,6 @@
from datetime import datetime, date, timedelta from datetime import datetime, date, timedelta
import unittest import unittest
import moment import moment
import moment_parse
# Helpful strftime() format that imcludes all parts of the date including the time zone. # Helpful strftime() format that imcludes all parts of the date including the time zone.
fmt = "%Y-%m-%d %H:%M:%S %Z" fmt = "%Y-%m-%d %H:%M:%S %Z"
@ -60,78 +59,6 @@ class TestMoment(unittest.TestCase):
[datetime(2037, 11, 1, 1, 0, 0, 0), 2140675200000, "PDT", 420, 1, 0], [datetime(2037, 11, 1, 1, 0, 0, 0), 2140675200000, "PDT", 420, 1, 0],
] ]
parse_samples = [
# Basic set
['MM-DD-YYYY', '12-02-1999', 944092800.000000],
['DD-MM-YYYY', '12-02-1999', 918777600.000000],
['DD/MM/YYYY', '12/02/1999', 918777600.000000],
['DD_MM_YYYY', '12_02_1999', 918777600.000000],
['DD:MM:YYYY', '12:02:1999', 918777600.000000],
['D-M-YY', '2-2-99', 917913600.000000],
['YY', '99', 922060800.000000],
['DD-MM-YYYY h:m:s', '12-02-1999 2:45:10', 918787510.000000],
['DD-MM-YYYY h:m:s a', '12-02-1999 2:45:10 am', 918787510.000000],
['DD-MM-YYYY h:m:s a', '12-02-1999 2:45:10 pm', 918830710.000000],
['h:mm a', '12:00 pm', 1458648000.000000],
['h:mm a', '12:30 pm', 1458649800.000000],
['h:mm a', '12:00 am', 1458604800.000000],
['h:mm a', '12:30 am', 1458606600.000000],
['HH:mm', '12:00', 1458648000.000000],
['YYYY-MM-DDTHH:mm:ss', '2011-11-11T11:11:11', 1321009871.000000],
['ddd MMM DD HH:mm:ss YYYY', 'Tue Apr 07 22:52:51 2009', 1239144771.000000],
['ddd MMMM DD HH:mm:ss YYYY', 'Tue April 07 22:52:51 2009', 1239144771.000000],
['HH:mm:ss', '12:00:00', 1458648000.000000],
['HH:mm:ss', '12:30:00', 1458649800.000000],
['HH:mm:ss', '00:00:00', 1458604800.000000],
['HH:mm:ss S', '00:30:00 1', 1458606600.100000],
['HH:mm:ss SS', '00:30:00 12', 1458606600.120000],
['HH:mm:ss SSS', '00:30:00 123', 1458606600.123000],
['HH:mm:ss S', '00:30:00 7', 1458606600.700000],
['HH:mm:ss SS', '00:30:00 78', 1458606600.780000],
['HH:mm:ss SSS', '00:30:00 789', 1458606600.789000],
# Dropped m
['MM/DD/YYYY h:m:s a', '05/1/2012 12:25:00 p', 1335875100.000000],
['MM/DD/YYYY h:m:s a', '05/1/2012 12:25:00 a', 1335831900.000000],
# 2 digit year with YYYY
['D/M/YYYY', '9/2/99', 918518400.000000],
['D/M/YYYY', '9/2/1999', 918518400.000000],
['D/M/YYYY', '9/2/66', -122860800.000000],
['D/M/YYYY', '9/2/65', 3001363200.000000],
# No separators
['MMDDYYYY', '12021999', 944092800.000000],
['DDMMYYYY', '12021999', 918777600.000000],
['YYYYMMDD', '19991202', 944092800.000000],
['DDMMMYYYY', '10Sep2001', 1000080000.000000],
# Error forgiveness
['MM/DD/YYYY', '12-02-1999', 944092800.000000],
['DD/MM/YYYY', '12/02 /1999', 918777600.000000],
['DD:MM:YYYY', '12:02 :1999', 918777600.000000],
['D-M-YY', '2 2 99', 917913600.000000],
['DD-MM-YYYY h:m:s', '12-02-1999 2:45:10.00', 918787510.000000],
['h:mm a', '12:00pm', 1458648000.000000],
['HH:mm', '1200', 1458648000.000000],
['dddd MMMM DD HH:mm:ss YYYY', 'Tue Apr 7 22:52:51 2009', 1239144771.000000],
['ddd MMM DD HH:mm:ss YYYY', 'Tuesday April 7 22:52:51 2009', 1239144771.000000],
['ddd MMM Do HH:mm:ss YYYY', 'Tuesday April 7th 22:52:51 2009', 1239144771.000000]
]
parse_timezone_samples = [
# Timezone corner cases
['MM-DD-YYYY h:ma', '3-13-2016 1:59am', 'America/New_York', 1457852340], # EST
['MM-DD-YYYY h:ma', '3-13-2016 2:00am', 'America/New_York', 1457848800], # Invalid, -1hr
['MM-DD-YYYY h:ma', '3-13-2016 2:59am', 'America/New_York', 1457852340], # Invalid, -1hr
['MM-DD-YYYY h:ma', '3-13-2016 3:00am', 'America/New_York', 1457852400], # EDT
['MM-DD-YYYY h:ma', '3-13-2016 1:59am', 'America/Los_Angeles', 1457863140], # PST
['MM-DD-YYYY h:ma', '3-13-2016 2:00am', 'America/Los_Angeles', 1457859600], # Invalid, -1hr
['MM-DD-YYYY h:ma', '3-13-2016 2:59am', 'America/Los_Angeles', 1457863140], # Invalid, -1hr
['MM-DD-YYYY h:ma', '3-13-2016 3:00am', 'America/Los_Angeles', 1457863200] # PDT
]
def assertMatches(self, data_entry, moment_obj): def assertMatches(self, data_entry, moment_obj):
date, timestamp, abbr, offset, hour, minute = data_entry date, timestamp, abbr, offset, hour, minute = data_entry
dt = moment_obj.datetime() dt = moment_obj.datetime()
@ -183,12 +110,6 @@ class TestMoment(unittest.TestCase):
self.assertEqual(dt.tzname(), abbr) self.assertEqual(dt.tzname(), abbr)
self.assertEqual(dt.utcoffset(), timedelta(minutes=-offset)) self.assertEqual(dt.utcoffset(), timedelta(minutes=-offset))
def test_parse(self):
for s in self.parse_samples:
self.assertEqual(moment_parse.parse(s[1], s[0], 'UTC', date(2016, 3, 22)), s[2])
for s in self.parse_timezone_samples:
self.assertEqual(moment_parse.parse(s[1], s[0], s[2], date(2016, 3, 22)), s[3])
def test_ts_to_dt(self): def test_ts_to_dt(self):
# Verify that ts_to_dt works as expected. # Verify that ts_to_dt works as expected.
value_sec = 1426291200 # 2015-03-14 00:00:00 in UTC value_sec = 1426291200 # 2015-03-14 00:00:00 in UTC

2
sandbox/grist/test_reflist_rel.py
View File

@ -64,7 +64,7 @@ class TestRefListRelation(test_engine.EngineTestCase):
self.apply_user_action( self.apply_user_action(
['AddColumn', 'TableC', 'gristHelper_Transform', { ['AddColumn', 'TableC', 'gristHelper_Transform', {
"type": 'Ref:TableA', "isFormula": True, "type": 'Ref:TableA', "isFormula": True,
"formula": "grist.Reference.typeConvert($ColB, TableA, 'ColA')", "visibleCol": 2, "formula": "TableA.lookupOne(ColA=$ColB)", "visibleCol": 2,
}]) }])
self.apply_user_action( self.apply_user_action(
['SetDisplayFormula', 'TableC', None, 7, '$gristHelper_Transform.ColA']) ['SetDisplayFormula', 'TableC', None, 7, '$gristHelper_Transform.ColA'])

31
sandbox/grist/useractions.py
View File

@ -1037,7 +1037,13 @@ class UserActions(object):
if not clean_colinfo["isFormula"]: if not clean_colinfo["isFormula"]:
raise ValueError("AddColumn: cannot add a non-formula column to a summary table") raise ValueError("AddColumn: cannot add a non-formula column to a summary table")
transform = col_id is not None and col_id.startswith('gristHelper_Transform') transform = (
col_id is not None and
col_id.startswith((
'gristHelper_Transform',
'gristHelper_Converted',
))
)
if transform: if transform:
# Delete any currently existing transform columns with the same id # Delete any currently existing transform columns with the same id
@ -1256,6 +1262,29 @@ class UserActions(object):
finally: finally:
self._engine.out_actions.undo.append(mod_action) self._engine.out_actions.undo.append(mod_action)
@useraction
def ConvertFromColumn(self, table_id, src_col_id, dst_col_id, typ, widgetOptions, visibleColRef):
from sandbox import call_external
table = self._engine.tables[table_id]
src_col = self._docmodel.get_column_rec(table_id, src_col_id)
src_column = table.get_column(src_col_id)
row_ids = list(table.row_ids)
src_values = [encode_object(src_column.raw_get(r)) for r in row_ids]
display_values = None
if src_col.displayCol:
display_col = table.get_column(src_col.displayCol.colId)
display_values = [encode_object(display_col.raw_get(r)) for r in row_ids]
converted_values = call_external(
"convertFromColumn",
src_col.id,
typ,
widgetOptions,
visibleColRef,
src_values,
display_values,
)
self.ModifyColumn(table_id, dst_col_id, {"type": typ})
self.BulkUpdateRecord(table_id, row_ids, {dst_col_id: converted_values})
@useraction @useraction
def CopyFromColumn(self, table_id, src_col_id, dst_col_id, widgetOptions): def CopyFromColumn(self, table_id, src_col_id, dst_col_id, widgetOptions):

66
sandbox/grist/usertypes.py
View File

@ -138,17 +138,6 @@ class BaseColumnType(object):
return objtypes.safe_repr(value_to_convert) return objtypes.safe_repr(value_to_convert)
# This is a user-facing method, hence the camel-case naming, as for `lookupRecords` and such.
@classmethod
def typeConvert(cls, value):
"""
Convert a value from a different type to something that this type can accept, as when
explicitly converting a column type. Note that usual conversion (such as converting numbers to
strings or vice versa) will still apply to the returned value.
"""
return value
class Text(BaseColumnType): class Text(BaseColumnType):
""" """
Text is the type for a field holding string (text) data. Text is the type for a field holding string (text) data.
@ -180,18 +169,6 @@ class Text(BaseColumnType):
def is_right_type(cls, value): def is_right_type(cls, value):
return isinstance(value, (six.string_types, NoneType)) return isinstance(value, (six.string_types, NoneType))
@classmethod
def typeConvert(cls, value):
if value is None:
# When converting NULLs (that typically show up as a plain empty cell for Numeric or Date
# columns) to Text, it makes more sense to end up with a plain blank text cell.
return ''
elif isinstance(value, bool):
# Normalize True/False to true/false (Toggle columns use true/false).
return str(value).lower()
else:
return value
class Blob(BaseColumnType): class Blob(BaseColumnType):
""" """
@ -302,13 +279,6 @@ class Date(Numeric):
def is_right_type(cls, value): def is_right_type(cls, value):
return isinstance(value, (float, six.integer_types, NoneType)) return isinstance(value, (float, six.integer_types, NoneType))
@classmethod
def typeConvert(cls, value, date_format, timezone='UTC'): # pylint: disable=arguments-differ
# Note: the timezone argument is used in DateTime conversions, allows sharing this method.
try:
return moment.parse(value, date_format, timezone)
except Exception:
return value
class DateTime(Date): class DateTime(Date):
""" """
@ -370,21 +340,6 @@ class ChoiceList(BaseColumnType):
return value is None or (isinstance(value, (tuple, list)) and return value is None or (isinstance(value, (tuple, list)) and
all(isinstance(item, six.string_types) for item in value)) all(isinstance(item, six.string_types) for item in value))
@classmethod
def typeConvert(cls, value):
if value is None:
return value
if isinstance(value, six.string_types) and not value.startswith('['):
# Try to parse as CSV. If this doesn't work, we'll still try usual conversions later.
try:
tags = next(csv.reader([value]))
return tuple(t.strip() for t in tags if t.strip())
except Exception:
pass
if not isinstance(value, (tuple, list)):
value = [Choice.typeConvert(value)]
return value
@classmethod @classmethod
def toString(cls, value): def toString(cls, value):
if isinstance(value, (tuple, list)): if isinstance(value, (tuple, list)):
@ -458,13 +413,6 @@ class Reference(Id):
def typename(cls): def typename(cls):
return "Ref" return "Ref"
@classmethod
def typeConvert(cls, value, ref_table, visible_col=None): # pylint: disable=arguments-differ
if value and ref_table and visible_col:
return ref_table.lookupOne(**{visible_col: value}) or six.text_type(value)
else:
return value
class ReferenceList(BaseColumnType): class ReferenceList(BaseColumnType):
""" """
@ -500,15 +448,6 @@ class ReferenceList(BaseColumnType):
return value is None or (isinstance(value, list) and return value is None or (isinstance(value, list) and
all(Reference.is_right_type(val) for val in value)) all(Reference.is_right_type(val) for val in value))
@classmethod
def typeConvert(cls, value, ref_table, visible_col=None): # noqa # pylint: disable=arguments-differ
# TODO this is based on Reference.typeConvert.
# It doesn't make much sense as a conversion but I don't know what would
if value and ref_table and visible_col:
return ref_table.lookupRecords(**{visible_col: value}) or six.text_type(value)
else:
return value
class Attachments(ReferenceList): class Attachments(ReferenceList):
""" """
@ -516,8 +455,3 @@ class Attachments(ReferenceList):
""" """
def __init__(self): def __init__(self):
super(Attachments, self).__init__('_grist_Attachments') super(Attachments, self).__init__('_grist_Attachments')
@classmethod
def typeConvert(cls, value): # noqa # pylint: disable=arguments-differ
# Don't use ReferenceList.typeConvert which is called with a different number of arguments
return value

4
test/nbrowser/gristUtils.ts
View File

@ -1053,6 +1053,10 @@ export async function setType(type: RegExp, options: {skipWait?: boolean} = {})
if (!options.skipWait) { await waitForServer(); } if (!options.skipWait) { await waitForServer(); }
} }
export async function applyTypeTransform() {
await driver.findContent('.type_transform_prompt button', /Apply/).click();
}
export async function isMac(): Promise<boolean> { export async function isMac(): Promise<boolean> {
return /Darwin|Mac|iPod|iPhone|iPad/i.test((await driver.getCapabilities()).get('platform')); return /Darwin|Mac|iPod|iPhone|iPad/i.test((await driver.getCapabilities()).get('platform'));
} }