From 5d671bf0b30760d40b424035ab8afb6722cdd583 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Fri, 4 Feb 2022 13:13:03 +0200 Subject: [PATCH] (core) New type conversion in the backend Summary: This is https://phab.getgrist.com/D3205 plus some changes (https://github.com/dsagal/grist/compare/type-convert...type-convert-server?expand=1) that move the conversion process to the backend. A new user action ConvertFromColumn uses `call_external` so that the data engine can delegate back to ActiveDoc. Code for creating formatters and parsers is significantly refactored so that most of the logic is in `common` and can be used in different ways. Test Plan: The original diff adds plenty of tests. Reviewers: georgegevoian Reviewed By: georgegevoian Subscribers: dsagal Differential Revision: https://phab.getgrist.com/D3240 --- app/client/components/ColumnTransform.ts | 28 +- app/client/components/TypeConversion.ts | 107 +++----- app/client/components/TypeTransform.ts | 72 +++-- app/client/models/DocModel.ts | 2 +- app/client/models/entities/ColumnRec.ts | 52 ++-- app/client/models/entities/ViewFieldRec.ts | 9 +- app/client/ui/Pages.ts | 2 +- app/common/ValueConverter.ts | 255 ++++++++++++++++++ app/common/ValueFormatter.ts | 72 ++++- app/common/ValueParser.ts | 88 +++--- .../models => common}/isHiddenTable.ts | 6 +- app/server/lib/ActiveDoc.ts | 7 + app/server/lib/GranularAccess.ts | 1 + app/server/lib/ISandbox.ts | 3 + app/server/lib/NSandbox.ts | 3 +- sandbox/grist/column.py | 19 +- sandbox/grist/functions/info.py | 8 + sandbox/grist/functions/schedule.py | 6 +- sandbox/grist/moment.py | 6 - sandbox/grist/moment_parse.py | 161 ----------- sandbox/grist/test_moment.py | 79 ------ sandbox/grist/test_reflist_rel.py | 2 +- sandbox/grist/useractions.py | 31 ++- sandbox/grist/usertypes.py | 66 ----- test/nbrowser/gristUtils.ts | 4 + 25 files changed, 595 insertions(+), 494 deletions(-) create mode 100644 app/common/ValueConverter.ts rename app/{client/models => common}/isHiddenTable.ts (67%) delete mode 100644 sandbox/grist/moment_parse.py diff --git a/app/client/components/ColumnTransform.ts b/app/client/components/ColumnTransform.ts index 6a00bc4d..28069c4f 100644 --- a/app/client/components/ColumnTransform.ts +++ b/app/client/components/ColumnTransform.ts @@ -142,6 +142,10 @@ export class ColumnTransform extends Disposable { return actions.every(action => ( // ['AddColumn', USER_TABLE, 'gristHelper_Transform', colInfo] (action[2] === 'gristHelper_Transform') || + // ['AddColumn', USER_TABLE, 'gristHelper_Converted', colInfo] + (action[2] === 'gristHelper_Converted') || + // ['ConvertFromColumn', USER_TABLE, SOURCE_COLUMN, 'gristHelper_Converted'] + (action[3] === 'gristHelper_Converted') || // ["SetDisplayFormula", USER_TABLE, ...] (action[0] === 'SetDisplayFormula') || // ['UpdateRecord', '_grist_Table_column', transformColId, ...] @@ -192,7 +196,6 @@ export class ColumnTransform extends Disposable { // Define variables used after await, since this will be disposed by then. const transformColId = this.transformColumn.colId(); const field = this.field; - const fieldBuilder = this._fieldBuilder; const origRef = this.origColumn.getRowId(); const tableData = this._tableData; this.isCallPending(true); @@ -201,17 +204,36 @@ export class ColumnTransform extends Disposable { // TODO: Values flicker during executing since transform column remains a formula as values are copied // back to the original column. The CopyFromColumn useraction really ought to be "CopyAndRemove" since // that seems the best way to avoid calculating the formula on wrong values. - return await tableData.sendTableAction(['CopyFromColumn', transformColId, this.origColumn.colId(), - JSON.stringify(fieldBuilder.options())]); + await this.gristDoc.docData.sendActions(this.executeActions()); } } finally { // Wait until the change completed to set column back, to avoid value flickering. field.colRef(origRef); void tableData.sendTableAction(['RemoveColumn', transformColId]); + this.cleanup(); this.dispose(); } } + /** + * The user actions to send when actually executing the transform. + */ + protected executeActions(): UserAction[] { + return [ + [ + 'CopyFromColumn', + this._tableData.tableId, + this.transformColumn.colId(), + this.origColumn.colId(), + JSON.stringify(this._fieldBuilder.options()), + ], + ]; + } + + protected cleanup() { + // For overriding + } + protected getIdentityFormula() { return 'return $' + this.origColumn.colId(); } diff --git a/app/client/components/TypeConversion.ts b/app/client/components/TypeConversion.ts index 5799af90..11cda7da 100644 --- a/app/client/components/TypeConversion.ts +++ b/app/client/components/TypeConversion.ts @@ -6,7 +6,6 @@ import {DocModel} from 'app/client/models/DocModel'; import {ColumnRec} from 'app/client/models/entities/ColumnRec'; -import * as UserType from 'app/client/widgets/UserType'; import * as gristTypes from 'app/common/gristTypes'; import {isFullReferencingType} from 'app/common/gristTypes'; import * as gutil from 'app/common/gutil'; @@ -90,7 +89,7 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe type: addColTypeSuffix(toTypeMaybeFull, origCol, docModel), isFormula: true, visibleCol: 0, - formula: "", // Will be filled in at the end. + formula: "CURRENT_CONVERSION(rec)", }; const prevOptions = origCol.widgetOptionsJson.peek() || {}; @@ -139,23 +138,35 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe case 'RefList': { // Set suggested destination table and visible column. - // Null if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen). - const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`)!; - - // Finds a reference suggestion column and sets it as the current reference value. - const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100); - if (!columnData) { break; } - columnData.delete(gristTypes.getDefaultForType(origCol.type())); - - // 'findColFromValues' function requires an array since it sends the values to the sandbox. - const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId); - const suggestedColRef = matches.find(match => match !== origCol.getRowId()); - if (!suggestedColRef) { break; } - const suggestedCol = docModel.columns.getRowModel(suggestedColRef); - const suggestedTableId = suggestedCol.table().tableId(); - if (optTableId && suggestedTableId !== optTableId) { - console.warn("Inappropriate column received from findColFromValues"); - break; + // Undefined if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen). + const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`) || undefined; + + let suggestedColRef: number; + let suggestedTableId: string; + const origColTypeInfo = gristTypes.extractInfoFromColType(origCol.type.peek()); + if (!optTableId && origColTypeInfo.type === "Ref" || origColTypeInfo.type === "RefList") { + // When converting between Ref and Reflist, initially suggest the same table and visible column. + // When converting, if the table is the same, it's a special case. + // The visible column will not affect conversion. + // It will simply wrap the reference (row ID) in a list or extract the one element of a reference list. + suggestedColRef = origCol.visibleCol.peek(); + suggestedTableId = origColTypeInfo.tableId; + } else { + // Finds a reference suggestion column and sets it as the current reference value. + const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100); + if (!columnData) { break; } + columnData.delete(gristTypes.getDefaultForType(origCol.type())); + + // 'findColFromValues' function requires an array since it sends the values to the sandbox. + const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId); + suggestedColRef = matches.find(match => match !== origCol.getRowId())!; + if (!suggestedColRef) { break; } + const suggestedCol = docModel.columns.getRowModel(suggestedColRef); + suggestedTableId = suggestedCol.table().tableId(); + if (optTableId && suggestedTableId !== optTableId) { + console.warn("Inappropriate column received from findColFromValues"); + break; + } } colInfo.type = `${toType}:${suggestedTableId}`; colInfo.visibleCol = suggestedColRef; @@ -163,11 +174,9 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe } } - const newOptions = UserType.mergeOptions(widgetOptions || {}, colInfo.type); if (widgetOptions) { colInfo.widgetOptions = JSON.stringify(widgetOptions); } - colInfo.formula = getDefaultFormula(docModel, origCol, colInfo.type, colInfo.visibleCol, newOptions); return colInfo; } @@ -184,62 +193,6 @@ export async function setDisplayFormula( } } -// Given the original column and info about the new column, returns the formula to use for the -// transform column to do the transformation. -export function getDefaultFormula( - docModel: DocModel, origCol: ColumnRec, newType: string, - newVisibleCol: number, newWidgetOptions: any): string { - - const colId = origCol.colId(); - const oldVisibleColName = isReferenceCol(origCol) ? - getVisibleColName(docModel, origCol.visibleCol()) : undefined; - - let origValFormula = oldVisibleColName ? - // The `str()` below converts AltText to plain text. - `($${colId}.${oldVisibleColName} - if ISREF($${colId}) or ISREFLIST($${colId}) - else str($${colId}))` - : `$${colId}`; - - if (origCol.type.peek() === 'ChoiceList') { - origValFormula = `grist.ChoiceList.toString($${colId})`; - } - - const toTypePure: string = gristTypes.extractTypeFromColType(newType); - - // The args are used to construct the call to grist.TYPE.typeConvert(value, [params]). - // Optional parameters depend on the type; see sandbox/grist/usertypes.py - const args: string[] = [origValFormula]; - switch (toTypePure) { - case 'Ref': - case 'RefList': - { - const table = gutil.removePrefix(newType, toTypePure + ":"); - args.push(table || 'None'); - const visibleColName = getVisibleColName(docModel, newVisibleCol); - if (visibleColName) { - args.push(q(visibleColName)); - } - break; - } - case 'Date': { - args.push(q(newWidgetOptions.dateFormat)); - break; - } - case 'DateTime': { - const timezone = gutil.removePrefix(newType, "DateTime:") || ''; - const format = newWidgetOptions.dateFormat + ' ' + newWidgetOptions.timeFormat; - args.push(q(format), q(timezone)); - break; - } - } - return `grist.${gristTypes.getGristType(toTypePure)}.typeConvert(${args.join(', ')})`; -} - -function q(value: string): string { - return "'" + value.replace(/'/g, "\\'") + "'"; -} - // Returns the name of the visibleCol given its rowId. function getVisibleColName(docModel: DocModel, visibleColRef: number): string|undefined { return visibleColRef ? docModel.columns.getRowModel(visibleColRef).colId() : undefined; diff --git a/app/client/components/TypeTransform.ts b/app/client/components/TypeTransform.ts index c7a92cce..6655d8e3 100644 --- a/app/client/components/TypeTransform.ts +++ b/app/client/components/TypeTransform.ts @@ -10,13 +10,12 @@ import {ColumnTransform} from 'app/client/components/ColumnTransform'; import {GristDoc} from 'app/client/components/GristDoc'; import * as TypeConversion from 'app/client/components/TypeConversion'; import {reportError} from 'app/client/models/errors'; -import * as modelUtil from 'app/client/models/modelUtil'; import {cssButtonRow} from 'app/client/ui/RightPanel'; import {basicButton, primaryButton} from 'app/client/ui2018/buttons'; import {testId} from 'app/client/ui2018/cssVars'; import {FieldBuilder} from 'app/client/widgets/FieldBuilder'; import {NewAbstractWidget} from 'app/client/widgets/NewAbstractWidget'; -import {ColValues} from 'app/common/DocActions'; +import {ColValues, UserAction} from 'app/common/DocActions'; import {Computed, dom, fromKo, Observable} from 'grainjs'; import isEmpty = require('lodash/isEmpty'); import pickBy = require('lodash/pickBy'); @@ -86,15 +85,6 @@ export class TypeTransform extends ColumnTransform { ); } - protected async resetToDefaultFormula() { - if (!this.isFinalizing()) { - const toType = this.transformColumn.type.peek(); - const formula = TypeConversion.getDefaultFormula(this.gristDoc.docModel, this.origColumn, - toType, this.field.visibleColRef(), this.field.widgetOptionsJson()); - await modelUtil.setSaveValue(this.transformColumn.formula, formula); - } - } - /** * Overrides parent method to initialize the transform column with guesses as to the particular * type and column options. @@ -103,20 +93,55 @@ export class TypeTransform extends ColumnTransform { protected async addTransformColumn(toType: string) { const docModel = this.gristDoc.docModel; const colInfo = await TypeConversion.prepTransformColInfo(docModel, this.origColumn, this.origDisplayCol, toType); - const newColInfo = await this._tableData.sendTableAction(['AddColumn', 'gristHelper_Transform', colInfo]); - const tcol = docModel.columns.getRowModel(newColInfo.colRef); - await TypeConversion.setDisplayFormula(docModel, tcol); - return newColInfo.colRef; + const newColInfos = await this._tableData.sendTableActions([ + ['AddColumn', 'gristHelper_Converted', {...colInfo, isFormula: false, formula: ''}], + ['AddColumn', 'gristHelper_Transform', colInfo], + ]); + const transformColRef = newColInfos[1].colRef; + this.transformColumn = docModel.columns.getRowModel(transformColRef); + await this.convertValues(); + return transformColRef; + } + + protected convertValuesActions(): UserAction[] { + const tableId = this._tableData.tableId; + const srcColId = this.origColumn.colId.peek(); + const dstColId = "gristHelper_Converted"; + const type = this.transformColumn.type.peek(); + const widgetOptions = this.transformColumn.widgetOptions.peek(); + const visibleColRef = this.transformColumn.visibleCol.peek(); + return [['ConvertFromColumn', tableId, srcColId, dstColId, type, widgetOptions, visibleColRef]]; + } + + protected async convertValues() { + await Promise.all([ + this.gristDoc.docData.sendActions(this.convertValuesActions()), + TypeConversion.setDisplayFormula(this.gristDoc.docModel, this.transformColumn), + ]); + } + + protected executeActions(): UserAction[] { + return [...this.convertValuesActions(), ...super.executeActions()]; } /** * Overrides parent method to subscribe to changes to the transform column. */ protected postAddTransformColumn() { - // When a user-initiated change is saved to type or widgetOptions, update the formula. - this.autoDispose(this.transformColumn.type.subscribe(this.resetToDefaultFormula, this, "save")); - this.autoDispose(this.transformColumn.visibleCol.subscribe(this.resetToDefaultFormula, this, "save")); - this.autoDispose(this.field.widgetOptionsJson.subscribe(this.resetToDefaultFormula, this, "save")); + // When a user-initiated change is saved to type or widgetOptions, reconvert the values + // Need to subscribe to both 'change' and 'save' for type which can come from setting the type itself + // or e.g. a change to DateTime timezone. + this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "change")); + this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "save")); + this.autoDispose(this.transformColumn.visibleCol.subscribe(this.convertValues, this, "save")); + this.autoDispose(this.field.widgetOptionsJson.subscribe(this.convertValues, this, "save")); + } + + /** + * Overrides parent method to delete extra column + */ + protected cleanup() { + void this._tableData.sendTableAction(['RemoveColumn', 'gristHelper_Converted']); } /** @@ -129,9 +154,10 @@ export class TypeTransform extends ColumnTransform { const tcol = this.transformColumn; const changedInfo = pickBy(colInfo, (val, key) => (val !== tcol[key as keyof TypeConversion.ColInfo].peek())); - return Promise.all([ - isEmpty(changedInfo) ? undefined : tcol.updateColValues(changedInfo as ColValues), - TypeConversion.setDisplayFormula(docModel, tcol, changedInfo.visibleCol) - ]); + if (!isEmpty(changedInfo)) { + // Update the transform column, particularly the type. + // This will trigger the subscription in postAddTransformColumn and lead to calling convertValues. + await tcol.updateColValues(changedInfo as ColValues); + } } } diff --git a/app/client/models/DocModel.ts b/app/client/models/DocModel.ts index c1d7eee2..dedb5315 100644 --- a/app/client/models/DocModel.ts +++ b/app/client/models/DocModel.ts @@ -22,7 +22,7 @@ import {urlState} from 'app/client/models/gristUrlState'; import * as MetaRowModel from 'app/client/models/MetaRowModel'; import * as MetaTableModel from 'app/client/models/MetaTableModel'; import * as rowset from 'app/client/models/rowset'; -import {isHiddenTable} from 'app/client/models/isHiddenTable'; +import {isHiddenTable} from 'app/common/isHiddenTable'; import {schema, SchemaTypes} from 'app/common/schema'; import {ACLRuleRec, createACLRuleRec} from 'app/client/models/entities/ACLRuleRec'; diff --git a/app/client/models/entities/ColumnRec.ts b/app/client/models/entities/ColumnRec.ts index 00198f34..53af467e 100644 --- a/app/client/models/entities/ColumnRec.ts +++ b/app/client/models/entities/ColumnRec.ts @@ -2,8 +2,13 @@ import {KoArray} from 'app/client/lib/koArray'; import {DocModel, IRowModel, recordSet, refRecord, TableRec, ViewFieldRec} from 'app/client/models/DocModel'; import {jsonObservable, ObjObservable} from 'app/client/models/modelUtil'; import * as gristTypes from 'app/common/gristTypes'; -import {getReferencedTableId, isFullReferencingType} from 'app/common/gristTypes'; -import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter'; +import {getReferencedTableId} from 'app/common/gristTypes'; +import { + BaseFormatter, + createFullFormatterRaw, + createVisibleColFormatterRaw, + FullFormatterArgs +} from 'app/common/ValueFormatter'; import * as ko from 'knockout'; // Represents a column in a user-defined table. @@ -124,38 +129,23 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void { // Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol // associated with this column. If no visible column available, return formatting for the column itself. - this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this, docModel)); + this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'vcol')); - this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, this.visibleColFormatter())); -} - -export function visibleColFormatterForRec( - rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel -): BaseFormatter { - const vcol = rec.visibleColModel(); - const documentSettings = docModel.docInfoRow.documentSettingsJson(); - const type = colRec.type(); - if (isFullReferencingType(type)) { - if (vcol.getRowId() === 0) { - // This column displays the Row ID, e.g. Table1[2] - // referencedTableId may actually be empty if the table is hidden - const referencedTableId: string = colRec.refTable()?.tableId() || ""; - return createFormatter('Id', {tableId: referencedTableId}, documentSettings); - } else { - return createFormatter(vcol.type(), vcol.widgetOptionsJson(), documentSettings); - } - } else { - // For non-reference columns, there's no 'visible column' and we just return a regular formatter - return createFormatter(type, rec.widgetOptionsJson(), documentSettings); - } + this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'full')); } export function formatterForRec( - rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, visibleColFormatter: BaseFormatter + rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, kind: 'full' | 'vcol' ): BaseFormatter { - const type = colRec.type(); - // Ref/RefList columns delegate most formatting to the visibleColFormatter - const widgetOpts = {...rec.widgetOptionsJson(), visibleColFormatter}; - const documentSettings = docModel.docInfoRow.documentSettingsJson(); - return createFormatter(type, widgetOpts, documentSettings); + const vcol = rec.visibleColModel(); + const func = kind === 'full' ? createFullFormatterRaw : createVisibleColFormatterRaw; + const args: FullFormatterArgs = { + docData: docModel.docData, + type: colRec.type(), + widgetOpts: rec.widgetOptionsJson(), + visibleColType: vcol?.type(), + visibleColWidgetOpts: vcol?.widgetOptionsJson(), + docSettings: docModel.docInfoRow.documentSettingsJson(), + }; + return func(args); } diff --git a/app/client/models/entities/ViewFieldRec.ts b/app/client/models/entities/ViewFieldRec.ts index eb7eafed..4652281b 100644 --- a/app/client/models/entities/ViewFieldRec.ts +++ b/app/client/models/entities/ViewFieldRec.ts @@ -1,5 +1,5 @@ import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel'; -import {formatterForRec, visibleColFormatterForRec} from 'app/client/models/entities/ColumnRec'; +import {formatterForRec} from 'app/client/models/entities/ColumnRec'; import * as modelUtil from 'app/client/models/modelUtil'; import * as UserType from 'app/client/widgets/UserType'; import {DocumentSettings} from 'app/common/DocumentSettings'; @@ -172,13 +172,14 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void // Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol // associated with this field. If no visible column available, return formatting for the field itself. - this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this.column(), docModel)); + this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'vcol')); - this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, this.visibleColFormatter())); + this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'full')); this.createValueParser = function() { const fieldRef = this.useColOptions.peek() ? undefined : this.id.peek(); - return createParser(docModel.docData, this.colRef.peek(), fieldRef); + const parser = createParser(docModel.docData, this.colRef.peek(), fieldRef); + return parser.cleanParse.bind(parser); }; // The widgetOptions to read and write: either the column's or the field's own. diff --git a/app/client/ui/Pages.ts b/app/client/ui/Pages.ts index 0b678f36..9411ffa8 100644 --- a/app/client/ui/Pages.ts +++ b/app/client/ui/Pages.ts @@ -3,7 +3,7 @@ import { duplicatePage } from "app/client/components/duplicatePage"; import { GristDoc } from "app/client/components/GristDoc"; import { PageRec } from "app/client/models/DocModel"; import { urlState } from "app/client/models/gristUrlState"; -import { isHiddenTable } from 'app/client/models/isHiddenTable'; +import { isHiddenTable } from 'app/common/isHiddenTable'; import * as MetaTableModel from "app/client/models/MetaTableModel"; import { find as findInTree, fromTableData, TreeItemRecord, TreeRecord, TreeTableData} from "app/client/models/TreeModel"; diff --git a/app/common/ValueConverter.ts b/app/common/ValueConverter.ts new file mode 100644 index 00000000..596a6a36 --- /dev/null +++ b/app/common/ValueConverter.ts @@ -0,0 +1,255 @@ +import {DocData} from 'app/common/DocData'; +import * as gristTypes from 'app/common/gristTypes'; +import {isList} from 'app/common/gristTypes'; +import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFormatter'; +import { + createParserOrFormatterArgumentsRaw, + createParserRaw, + ReferenceListParser, + ReferenceParser, + ValueParser +} from 'app/common/ValueParser'; +import {CellValue, GristObjCode} from 'app/plugin/GristData'; + + +/** + * Base class for converting values from one type to another with the convert() method. + * Has a formatter for the source column + * and a parser for the destination column. + * + * The default convert() is for non-list destination types, so if the source value + * is a list it only converts nicely if the list contains exactly one element. + */ +export class ValueConverter { + constructor(public formatter: BaseFormatter, public parser: ValueParser) { + } + + public convert(value: any): any { + if (isList(value)) { + if (value.length === 1) { + // Empty list: ['L'] + return null; + } else if (value.length === 2) { + // Singleton list: ['L', value] + // Convert just that one value. + value = value[1]; + } else { + // List with multiple values. Since we're converting to just one value, + // format the whole thing as text, which is an error for most types. + return this.formatter.formatAny(value); + } + } + return this.convertInner(value); + } + + protected convertInner(value: any): any { + const formatted = this.formatter.formatAny(value); + return this.parser.cleanParse(formatted); + } +} + +/** + * Base class for converting to a list type (Reference List or Choice List). + * + * Wraps single values in a list, and converts lists elementwise. + */ +class ListConverter extends ValueConverter { + // Don't parse strings like "Smith, John" which may look like lists but represent a single choice. + // TODO this works when the source is a Choice column, but not when it's a Reference to a Choice column. + // But the guessed choices are also broken in that case. + private _choices: Set = new Set((this.formatter.widgetOpts as any).choices || []); + + public convert(value: any): any { + if (typeof value === "string" && !this._choices.has(value)) { + // Parse CSV/JSON + return this.parser.cleanParse(value); + } + const values = isList(value) ? value.slice(1) : [value]; + if (!values.length || value == null) { + return null; + } + return this.handleValues(value, values.map(v => this.convertInner(v))); + } + + protected handleValues(originalValue: any, values: any[]) { + return ['L', ...values]; + } +} + +class ChoiceListConverter extends ListConverter { + /** + * Convert each source value to a 'Choice' + */ + protected convertInner(value: any): any { + return this.formatter.formatAny(value); + } +} + +class ReferenceListConverter extends ListConverter { + private _innerConverter = new ReferenceConverter( + this.formatter, + new ReferenceParser("Ref", this.parser.widgetOpts, this.parser.docSettings), + ); + + constructor(public formatter: BaseFormatter, public parser: ReferenceListParser) { + super(formatter, parser); + // Prevent the parser from looking up reference values in the frontend. + // Leave it to the data engine which has a much more efficient algorithm for long lists of values. + delete parser.tableData; + } + + public handleValues(originalValue: any, values: any[]): any { + const result = []; + let lookupColumn: string = ""; + const raw = this.formatter.formatAny(originalValue); // AltText if the reference lookup fails + for (const value of values) { + if (typeof value === "string") { + // Failed to parse one of the references, so return a raw string for the whole thing + return raw; + } else { + // value is a lookup tuple: ['l', value, options] + result.push(value[1]); + lookupColumn = value[2].column; + } + } + return ['l', result, {column: lookupColumn, raw}]; + } + + /** + * Convert each source value to a 'Reference' + */ + protected convertInner(value: any): any { + return this._innerConverter.convert(value); + } +} + +class ReferenceConverter extends ValueConverter { + private _innerConverter: ValueConverter = createConverter(this.formatter, this.parser.visibleColParser); + + constructor(public formatter: BaseFormatter, public parser: ReferenceParser) { + super(formatter, parser); + // Prevent the parser from looking up reference values in the frontend. + // Leave it to the data engine which has a much more efficient algorithm for long lists of values. + delete parser.tableData; + } + + protected convertInner(value: any): any { + // Convert to the type of the visible column. + const converted = this._innerConverter.convert(value); + return this.parser.lookup(converted, this.formatter.formatAny(value)); + } +} + +class NumericConverter extends ValueConverter { + protected convertInner(value: any): any { + if (typeof value === "boolean") { + return value ? 1 : 0; + } + return super.convertInner(value); + } +} + +class DateConverter extends ValueConverter { + private _sourceType = gristTypes.extractInfoFromColType(this.formatter.type); + + protected convertInner(value: any): any { + // When converting Date->DateTime, DateTime->Date, or between DateTime timezones, + // it's important to send an encoded Date/DateTime object rather than just a timestamp number + // so that the data engine knows what to do in do_convert, especially regarding timezones. + // If the source column is a Reference to a Date/DateTime then `value` is already + // an encoded object from the display column which has type Any. + value = gristTypes.reencodeAsAny(value, this._sourceType); + if (Array.isArray(value) && ( + value[0] === GristObjCode.Date || + value[0] === GristObjCode.DateTime + )) { + return value; + } + return super.convertInner(value); + } +} + +export const valueConverterClasses: { [type: string]: typeof ValueConverter } = { + Date: DateConverter, + DateTime: DateConverter, + ChoiceList: ChoiceListConverter, + Ref: ReferenceConverter, + RefList: ReferenceListConverter, + Numeric: NumericConverter, + Int: NumericConverter, +}; + +export function createConverter(formatter: BaseFormatter, parser: ValueParser) { + const cls = valueConverterClasses[gristTypes.extractTypeFromColType(parser.type)] || ValueConverter; + return new cls(formatter, parser); +} + +/** + * Used by the ConvertFromColumn user action in the data engine. + * The higher order function separates docData (passed by ActiveDoc) + * from the arguments passed to call_external in Python. + */ +export function convertFromColumn(docData: DocData) { + return function( + sourceColRef: number, + type: string, + widgetOpts: string, + visibleColRef: number, + values: ReadonlyArray, + displayColValues?: ReadonlyArray, + ): CellValue[] { + const formatter = createFullFormatterFromDocData(docData, sourceColRef); + const parser = createParserRaw( + ...createParserOrFormatterArgumentsRaw(docData, type, widgetOpts, visibleColRef) + ); + const converter = createConverter(formatter, parser); + return convertValues(converter, values, displayColValues || values); + }; +} + +export function convertValues( + converter: ValueConverter, + // Raw values from the actual column, e.g. row IDs for reference columns + values: ReadonlyArray, + // Values from the display column, which is the same as the raw values for non-referencing columns. + // In almost all cases these are the values that actually matter and get converted. + displayColValues: ReadonlyArray, +): CellValue[] { + // Converting Ref <-> RefList without changing the target table is a special case - see prepTransformColInfo. + // In this case we deal with the actual row IDs stored in the real column, + // whereas in all other cases we use display column values. + const sourceType = gristTypes.extractInfoFromColType(converter.formatter.type); + const targetType = gristTypes.extractInfoFromColType(converter.parser.type); + const refToRefList = ( + sourceType.type === "Ref" && + targetType.type === "RefList" && + sourceType.tableId === targetType.tableId + ); + const refListToRef = ( + sourceType.type === "RefList" && + targetType.type === "Ref" && + sourceType.tableId === targetType.tableId + ); + + return displayColValues.map((displayVal, i) => { + const actualValue = values[i]; + + if (refToRefList && typeof actualValue === "number") { + if (actualValue === 0) { + return null; + } else { + return ["L", actualValue]; + } + } else if (refListToRef && isList(actualValue)) { + if (actualValue.length === 1) { + // Empty list: ['L'] + return 0; + } else if (actualValue.length === 2) { + // Singleton list: ['L', rowId] + return actualValue[1]; + } + } + + return converter.convert(displayVal); + }); +} diff --git a/app/common/ValueFormatter.ts b/app/common/ValueFormatter.ts index b439cd1b..2f9b4efe 100644 --- a/app/common/ValueFormatter.ts +++ b/app/common/ValueFormatter.ts @@ -2,11 +2,14 @@ import {csvEncodeRow} from 'app/common/csvFormat'; import {CellValue} from 'app/common/DocActions'; +import {DocData} from 'app/common/DocData'; import {DocumentSettings} from 'app/common/DocumentSettings'; -import {getReferencedTableId, isList} from 'app/common/gristTypes'; import * as gristTypes from 'app/common/gristTypes'; +import {getReferencedTableId, isList} from 'app/common/gristTypes'; import * as gutil from 'app/common/gutil'; +import {isHiddenTable} from 'app/common/isHiddenTable'; import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat'; +import {createParserOrFormatterArguments, ReferenceParsingOptions} from 'app/common/ValueParser'; import {GristObjCode} from 'app/plugin/GristData'; import {decodeObject, GristDateTime} from 'app/plugin/objtypes'; import * as moment from 'moment-timezone'; @@ -280,3 +283,70 @@ export function createFormatter(type: string, widgetOpts: FormatOptions, docSett const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter; return new ctor(type, widgetOpts, docSettings); } + +export interface FullFormatterArgs { + docData: DocData; + type: string; + widgetOpts: FormatOptions; + visibleColType: string; + visibleColWidgetOpts: FormatOptions; + docSettings: DocumentSettings; +} + +/** + * Returns a constructor + * with a format function that can properly convert a value passed to it into the + * right format for that column. + * + * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field + * instead of the table column. + */ +export function createFullFormatterFromDocData( + docData: DocData, + colRef: number, + fieldRef?: number, +): BaseFormatter { + const [type, widgetOpts, docSettings] = createParserOrFormatterArguments(docData, colRef, fieldRef); + const {visibleColType, visibleColWidgetOpts} = widgetOpts as ReferenceParsingOptions; + return createFullFormatterRaw({ + docData, + type, + widgetOpts, + visibleColType, + visibleColWidgetOpts, + docSettings, + }); +} + +export function createFullFormatterRaw(args: FullFormatterArgs) { + const {type, widgetOpts, docSettings} = args; + const visibleColFormatter = createVisibleColFormatterRaw(args); + return createFormatter(type, {...widgetOpts, visibleColFormatter}, docSettings); +} + +export function createVisibleColFormatterRaw( + { + docData, + docSettings, + type, + visibleColType, + visibleColWidgetOpts, + widgetOpts + }: FullFormatterArgs +): BaseFormatter { + let referencedTableId = gristTypes.getReferencedTableId(type); + if (!referencedTableId) { + return createFormatter(type, widgetOpts, docSettings); + } else if (visibleColType) { + return createFormatter(visibleColType, visibleColWidgetOpts, docSettings); + } else { + // This column displays the Row ID, e.g. Table1[2] + // Make referencedTableId empty if the table is hidden + const tablesData = docData.getMetaTable("_grist_Tables"); + const tableRef = tablesData.findRow("tableId", referencedTableId); + if (isHiddenTable(tablesData, tableRef)) { + referencedTableId = ""; + } + return createFormatter('Id', {tableId: referencedTableId}, docSettings); + } +} diff --git a/app/common/ValueParser.ts b/app/common/ValueParser.ts index 8b71ec7d..df473142 100644 --- a/app/common/ValueParser.ts +++ b/app/common/ValueParser.ts @@ -33,6 +33,9 @@ export class ValueParser { } +class IdentityParser extends ValueParser { +} + /** * Same as basic Value parser, but will return null if a value is an empty string. */ @@ -117,7 +120,7 @@ class ChoiceListParser extends ValueParser { * stored on the field. These have to be specially derived * for referencing columns. See createParser. */ -interface ReferenceParsingOptions { +export interface ReferenceParsingOptions { visibleColId: string; visibleColType: string; visibleColWidgetOpts: FormatOptions; @@ -129,18 +132,22 @@ interface ReferenceParsingOptions { export class ReferenceParser extends ValueParser { public widgetOpts: ReferenceParsingOptions; - - protected _visibleColId = this.widgetOpts.visibleColId; - protected _tableData = this.widgetOpts.tableData; - protected _visibleColParser = createParserRaw( + public tableData = this.widgetOpts.tableData; + public visibleColParser = createParserRaw( this.widgetOpts.visibleColType, this.widgetOpts.visibleColWidgetOpts, this.docSettings, ); + protected _visibleColId = this.widgetOpts.visibleColId; + public parse(raw: string): any { - let value = this._visibleColParser(raw); - if (!value || !raw) { + const value = this.visibleColParser.cleanParse(raw); + return this.lookup(value, raw); + } + + public lookup(value: any, raw: string): any { + if (value == null || value === "" || !raw) { return 0; // default value for a reference column } @@ -154,7 +161,7 @@ export class ReferenceParser extends ValueParser { } } - if (!this._tableData?.isLoaded) { + if (!this.tableData?.isLoaded) { const options: { column: string, raw?: string } = {column: this._visibleColId}; if (value !== raw) { options.raw = raw; @@ -162,7 +169,7 @@ export class ReferenceParser extends ValueParser { return ['l', value, options]; } - return this._tableData.findMatchingRowId({[this._visibleColId]: value}) || raw; + return this.tableData.findMatchingRowId({[this._visibleColId]: value}) || raw; } } @@ -178,7 +185,7 @@ export class ReferenceListParser extends ReferenceParser { // csvDecodeRow should never raise an exception values = csvDecodeRow(raw); } - values = values.map(v => typeof v === "string" ? this._visibleColParser(v) : encodeObject(v)); + values = values.map(v => typeof v === "string" ? this.visibleColParser.cleanParse(v) : encodeObject(v)); if (!values.length || !raw) { return null; // null is the default value for a reference list column @@ -194,7 +201,7 @@ export class ReferenceListParser extends ReferenceParser { } } - if (!this._tableData?.isLoaded) { + if (!this.tableData?.isLoaded) { const options: { column: string, raw?: string } = {column: this._visibleColId}; if (!(values.length === 1 && values[0] === raw)) { options.raw = raw; @@ -204,7 +211,7 @@ export class ReferenceListParser extends ReferenceParser { const rowIds: number[] = []; for (const value of values) { - const rowId = this._tableData.findMatchingRowId({[this._visibleColId]: value}); + const rowId = this.tableData.findMatchingRowId({[this._visibleColId]: value}); if (rowId) { rowIds.push(rowId); } else { @@ -228,27 +235,21 @@ export const valueParserClasses: { [type: string]: typeof ValueParser } = { RefList: ReferenceListParser, }; -const identity = (value: string) => value; - /** - * Returns a function which can parse strings into values appropriate for + * Returns a ValueParser which can parse strings into values appropriate for * a specific widget field or table column. * widgetOpts is usually the field/column's widgetOptions JSON * but referencing columns need more than that, see ReferenceParsingOptions above. */ export function createParserRaw( type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings -): (value: string) => any { - const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)]; - if (cls) { - const parser = new cls(type, widgetOpts, docSettings); - return parser.cleanParse.bind(parser); - } - return identity; +): ValueParser { + const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)] || IdentityParser; + return new cls(type, widgetOpts, docSettings); } /** - * Returns a function which can parse strings into values appropriate for + * Returns a ValueParser which can parse strings into values appropriate for * a specific widget field or table column. * * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field @@ -258,23 +259,46 @@ export function createParser( docData: DocData, colRef: number, fieldRef?: number, -): (value: string) => any { +): ValueParser { + return createParserRaw(...createParserOrFormatterArguments(docData, colRef, fieldRef)); +} + +/** + * Returns arguments suitable for createParserRaw or createFormatter. Only for internal use. + * + * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field + * instead of the table column. + */ +export function createParserOrFormatterArguments( + docData: DocData, + colRef: number, + fieldRef?: number, +): [string, object, DocumentSettings] { const columnsTable = docData.getMetaTable('_grist_Tables_column'); const fieldsTable = docData.getMetaTable('_grist_Views_section_field'); - const docInfoTable = docData.getMetaTable('_grist_DocInfo'); const col = columnsTable.getRecord(colRef)!; - let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col; if (fieldRef) { fieldOrCol = fieldsTable.getRecord(fieldRef) || col; } - const widgetOpts = safeJsonParse(fieldOrCol.widgetOptions, {}); + return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol); +} + +export function createParserOrFormatterArgumentsRaw( + docData: DocData, + type: string, + widgetOptions: string, + visibleColRef: number, +): [string, object, DocumentSettings] { + const columnsTable = docData.getMetaTable('_grist_Tables_column'); + const docInfoTable = docData.getMetaTable('_grist_DocInfo'); + + const widgetOpts = safeJsonParse(widgetOptions, {}); - const type = col.type; if (isFullReferencingType(type)) { - const vcol = columnsTable.getRecord(fieldOrCol.visibleCol); + const vcol = columnsTable.getRecord(visibleColRef); widgetOpts.visibleColId = vcol?.colId || 'id'; widgetOpts.visibleColType = vcol?.type; widgetOpts.visibleColWidgetOpts = safeJsonParse(vcol?.widgetOptions || '', {}); @@ -284,7 +308,7 @@ export function createParser( const docInfo = docInfoTable.getRecord(1); const docSettings = safeJsonParse(docInfo!.documentSettings, {}) as DocumentSettings; - return createParserRaw(type, widgetOpts, docSettings); + return [type, widgetOpts, docSettings]; } /** @@ -311,12 +335,12 @@ function parseColValues( const parser = createParser(docData, colRef); // Optimisation: If there's no special parser for this column type, do nothing - if (parser === identity) { + if (parser instanceof IdentityParser) { return values; } function parseIfString(val: any) { - return typeof val === "string" ? parser(val) : val; + return typeof val === "string" ? parser.cleanParse(val) : val; } if (bulk) { diff --git a/app/client/models/isHiddenTable.ts b/app/common/isHiddenTable.ts similarity index 67% rename from app/client/models/isHiddenTable.ts rename to app/common/isHiddenTable.ts index 20b285b2..b40bf387 100644 --- a/app/client/models/isHiddenTable.ts +++ b/app/common/isHiddenTable.ts @@ -1,11 +1,11 @@ -import {RowId} from 'app/client/models/rowset'; -import {TableData} from 'app/client/models/TableData'; +import {UIRowId} from 'app/common/UIRowId'; +import {TableData} from "./TableData"; /** * Return whether a table identified by the rowId of its metadata record, should normally be * hidden from the user (e.g. as an option in the page-widget picker). */ -export function isHiddenTable(tablesData: TableData, tableRef: RowId): boolean { +export function isHiddenTable(tablesData: TableData, tableRef: UIRowId): boolean { const tableId = tablesData.getValue(tableRef, 'tableId') as string|undefined; return tablesData.getValue(tableRef, 'summarySourceTable') !== 0 || Boolean(tableId?.startsWith('GristHidden')); diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 447f1d06..28e1e289 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -41,6 +41,7 @@ import {schema, SCHEMA_VERSION} from 'app/common/schema'; import {MetaRowRecord} from 'app/common/TableData'; import {FetchUrlOptions, UploadResult} from 'app/common/uploads'; import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI'; +import {convertFromColumn} from 'app/common/ValueConverter'; import {parseUserAction} from 'app/common/ValueParser'; import {ParseOptions} from 'app/plugin/FileParserAPI'; import {GristDocAPI} from 'app/plugin/GristAPI'; @@ -1717,6 +1718,12 @@ export class ActiveDoc extends EventEmitter { logTimes: true, logMeta: {docId: this._docName}, preferredPythonVersion, + sandboxOptions: { + exports: { + convertFromColumn: (...args: Parameters>) => + convertFromColumn(this.docData!)(...args) + } + }, }); } } diff --git a/app/server/lib/GranularAccess.ts b/app/server/lib/GranularAccess.ts index 0bdbce02..321f92ba 100644 --- a/app/server/lib/GranularAccess.ts +++ b/app/server/lib/GranularAccess.ts @@ -87,6 +87,7 @@ const SPECIAL_ACTIONS = new Set(['InitNewDoc', 'TransformAndFinishImport', 'AddView', 'CopyFromColumn', + 'ConvertFromColumn', 'AddHiddenColumn', ]); diff --git a/app/server/lib/ISandbox.ts b/app/server/lib/ISandbox.ts index 4083f267..b0b95374 100644 --- a/app/server/lib/ISandbox.ts +++ b/app/server/lib/ISandbox.ts @@ -1,4 +1,5 @@ import * as log from 'app/server/lib/log'; +import {ISandboxOptions} from 'app/server/lib/NSandbox'; /** * Starting to whittle down the options used when creating a sandbox, to leave more @@ -17,6 +18,8 @@ export interface ISandboxCreationOptions { importMount?: string; // if defined, make this path available read-only as "/importdir" preferredPythonVersion?: '2' | '3'; + + sandboxOptions?: Partial; } export interface ISandbox { diff --git a/app/server/lib/NSandbox.ts b/app/server/lib/NSandbox.ts index 5ccc5a02..94cca548 100644 --- a/app/server/lib/NSandbox.ts +++ b/app/server/lib/NSandbox.ts @@ -33,7 +33,7 @@ type SandboxMethod = (...args: any[]) => any; * started by setting `useGristEntrypoint` (the only exception is * in tests) which runs grist/main.py. */ -interface ISandboxOptions { +export interface ISandboxOptions { command?: string; // External program or container to call to run the sandbox. args: string[]; // The arguments to pass to the python process. @@ -404,6 +404,7 @@ export class NSandboxCreator implements ISandboxCreator { preferredPythonVersion: this._preferredPythonVersion || options.preferredPythonVersion, useGristEntrypoint: true, importDir: options.importMount, + ...options.sandboxOptions, }; return new NSandbox(translatedOptions, spawners[this._flavor]); } diff --git a/sandbox/grist/column.py b/sandbox/grist/column.py index a26bb2ed..273bb52d 100644 --- a/sandbox/grist/column.py +++ b/sandbox/grist/column.py @@ -473,8 +473,23 @@ class BaseReferenceColumn(BaseColumn): .get_column_rec(self.table_id, self.col_id).visibleCol.colId or "id" ) - target_value = self._target_table.get_column(col_id)._convert_raw_value(value) - return self._target_table.lookup_one_record(**{col_id: target_value}) + column = self._target_table.get_column(col_id) + # `value` is an object encoded for transmission from JS to Python, + # which is decoded to `decoded_value`. + # `raw_value` is the kind of value that would be stored in `column`. + # `rich_value` is the type of value used in formulas, especially with `lookupRecords`. + # For example, for a Date column, `raw_value` is a numerical timestamp + # and `rich_value` is a `datetime.date` object, + # assuming `value` isn't of an invalid type. + # However `value` could either be just a number + # (in which case `decoded_value` would be a number as well) + # or an encoded date (or even datetime) object like ['d', number] + # (in which case `decoded_value` would be a `datetime.date` object, + # which would get converted back to a number and then back to a date object again!) + decoded_value = objtypes.decode_object(value) + raw_value = column.convert(decoded_value) + rich_value = column._convert_raw_value(raw_value) + return self._target_table.lookup_one_record(**{col_id: rich_value}) class ReferenceColumn(BaseReferenceColumn): diff --git a/sandbox/grist/functions/info.py b/sandbox/grist/functions/info.py index ae8883ad..b168e90f 100644 --- a/sandbox/grist/functions/info.py +++ b/sandbox/grist/functions/info.py @@ -500,6 +500,14 @@ def N(value): return 0 +def CURRENT_CONVERSION(rec): + """ + Special function used only when changing the type of a column. + Doesn't work in normal formulas. + """ + return rec.gristHelper_Converted + + def NA(): """ Returns the "value not available" error, `#N/A`. diff --git a/sandbox/grist/functions/schedule.py b/sandbox/grist/functions/schedule.py index b66a187a..46d9f500 100644 --- a/sandbox/grist/functions/schedule.py +++ b/sandbox/grist/functions/schedule.py @@ -1,12 +1,16 @@ from datetime import datetime, timedelta import re from .date import DATEADD, NOW, DTIME -from moment_parse import MONTH_NAMES, DAY_NAMES # Limit exports to schedule, so that upper-case constants like MONTH_NAMES, DAY_NAMES don't end up # exposed as if Excel-style functions (or break docs generation). __all__ = ['SCHEDULE'] +MONTH_NAMES = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', + 'september', 'october', 'november', 'december'] +# Regex list of lowercase weekdays with characters after the first three made optional +DAY_NAMES = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'] + def SCHEDULE(schedule, start=None, count=10, end=None): """ Returns the list of `datetime` objects generated according to the `schedule` string. Starts at diff --git a/sandbox/grist/moment.py b/sandbox/grist/moment.py index b3fcf8f7..d7cb3fa5 100644 --- a/sandbox/grist/moment.py +++ b/sandbox/grist/moment.py @@ -4,7 +4,6 @@ import marshal from time import time import bisect import os -import moment_parse import iso8601 import six from six.moves import zip @@ -13,7 +12,6 @@ from six.moves import zip ZoneRecord = namedtuple("ZoneRecord", ("name", "abbrs", "offsets", "untils")) # moment.py mirrors core functionality of moment-timezone.js -# moment.py includes function parse, located and documented in moment_parse.py # Documentation: http://momentjs.com/timezone/docs/ EPOCH = datetime(1970, 1, 1) @@ -67,10 +65,6 @@ def date_to_ts(date, timezone=None): ts = (date - DATE_EPOCH).total_seconds() return ts if not timezone else ts - timezone.offset(ts * 1000).total_seconds() -# Calls parse from moment_parse.py -def parse(date_string, parse_format, zonelabel='UTC'): - return moment_parse.parse(date_string, parse_format, zonelabel) - # Parses a datetime in the ISO format, YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM. Most parts are optional; # see https://pypi.org/project/iso8601/ for details. Returns a timestamp in seconds. def parse_iso(date_string, timezone=None): diff --git a/sandbox/grist/moment_parse.py b/sandbox/grist/moment_parse.py deleted file mode 100644 index c6a5cdc8..00000000 --- a/sandbox/grist/moment_parse.py +++ /dev/null @@ -1,161 +0,0 @@ -import re -from collections import OrderedDict -from datetime import datetime -import moment - -# Regex list of lowercase months with characters after the first three made optional -MONTH_NAMES = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', - 'september', 'october', 'november', 'december'] -MONTHS = [m[:3]+"(?:"+m[3:]+")?" if len(m) > 3 else m[:3] for m in MONTH_NAMES] -# Regex list of lowercase weekdays with characters after the first three made optional -DAY_NAMES = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'] -WEEKDAYS = [d[:3]+"(?:"+d[3:]+")?" for d in DAY_NAMES] - -# Acceptable format tokens mapped to what they should match in the date string -# Ordered so that larger configurations are matched first -DATE_TOKENS = OrderedDict([ - ("HH", r"(?P\d{1,2})"), # 24 hr - ("H", r"(?P\d{1,2})"), - ("hh", r"(?P\d{1,2})"), # 12 hr - ("h", r"(?P\d{1,2})"), - ("mm", r"(?P\d{1,2})"), # min - ("m", r"(?P\d{1,2})"), - ("A", r"(?P[ap]m?)"), # am/pm - ("a", r"(?P[ap]m?)"), - ("ss", r"(?P\d{1,2})"), # sec - ("s", r"(?P\d{1,2})"), - ("SSSSSS", r"(?P\d{1,6})"), # fractional second - ("SSSSS", r"(?P\d{1,6})"), - ("SSSS", r"(?P\d{1,6})"), - ("SSS", r"(?P\d{1,6})"), - ("SS", r"(?P\d{1,6})"), - ("S", r"(?P\d{1,6})"), - ("YYYY", r"(?P\d{4}|\d{2})"), # 4 or 2 digit year - ("YY", r"(?P\d{2})"), # 2 digit year - ("MMMM", r"(?P" + ("|".join(MONTHS)) + ")"), # month name, abbr or not - ("MMM", r"(?P" + ("|".join(MONTHS)) + ")"), - ("MM", r"(?P\d{1,2})"), # month num - ("M", r"(?P\d{1,2})"), - ("DD", r"(?P\d{1,2})"), # day num - ("Do", r"(?P\d{1,2})(st|nd|rd|th)"), - ("D", r"(?P\d{1,2})"), - ("dddd", r"(" + ("|".join(WEEKDAYS)) + ")"), # day name, abbr or not (ignored) - ("ddd", r"(" + ("|".join(WEEKDAYS)) + ")") -]) -DATE_TOKENS_REGEX = re.compile("("+("|".join(DATE_TOKENS))+")") - -# List of separators to replace and match any standard date/time separators -SEP = r"[\s/.\-:,]*" -SEP_REGEX = re.compile(SEP) -SEP_REPLACEMENT = SEP.replace("\\", "\\\\") - -# Maps date parse format to compile regex -FORMAT_CACHE = {} - -# Parses date_string using parse_format in the style of moment.js -# See: http://momentjs.com/docs/#/parsing -# Supports the following tokens: -# H HH 0..23 24 hour time -# h hh 1..12 12 hour time used with a A. -# a A am pm Post or ante meridiem -# m mm 0..59 Minutes -# s ss 0..59 Seconds -# S SS SSS 0..999 Fractional seconds -# YYYY 2014 4 or 2 digit year -# YY 14 2 digit year -# M MM 1..12 Month number -# MMM MMMM Jan..December Month name in locale set by moment.locale() -# D DD 1..31 Day of month -# Do 1st..31st Day of month with ordinal -def parse(date_string, parse_format, zonelabel='UTC', override_current_date=None): - """Parse a date string via a moment.js style parse format and a timezone string. - Supported tokens are documented above. Returns seconds since epoch""" - - if parse_format in FORMAT_CACHE: - # Check if parse_format has been cache, and retrieve if so - parser = FORMAT_CACHE[parse_format] - else: - # e.g. "MM-YY" -> "(?P\d{1,2})-(?P\d{2})" - # Note that DATE_TOKENS is ordered so that the longer letter chains are recognized first - tokens = DATE_TOKENS_REGEX.split(parse_format) - tokens = [DATE_TOKENS[t] if t in DATE_TOKENS else SEP_REGEX.sub(SEP_REPLACEMENT, t) - for t in tokens] - - # Compile new token string ignoring case (for month names) - parser = re.compile(''.join(tokens), re.I) - FORMAT_CACHE[parse_format] = parser - - match = parser.match(date_string) - - # Throw error if matching failed - if match is None: - raise Exception("Failed to parse %s with %s" % (date_string, parse_format)) - - # Create datetime from the results of parsing - current_date = override_current_date or moment.CURRENT_DATE - m = match.groupdict() - dt = datetime( - year=getYear(m, current_date.year), - month=getMonth(m, current_date.month), - day=int(m['D']) if ('D' in m) else current_date.day, - hour=getHour(m), - minute=int(m['m']) if ('m' in m) else 0, - second=int(m['s']) if ('s' in m) else 0, - microsecond=getMicrosecond(m) - ) - - # Parses the datetime with the given timezone to return the seconds since EPOCH - return moment.tz(dt, zonelabel).timestamp_s() - - -def getYear(match_dict, current_year): - if 'YYYY' in match_dict: - return int(match_dict['YYYY']) - elif 'YY' in match_dict: - match = match_dict['YY'] - if len(match) == 2: - # Must guess on the century, choose so the result is closest to the current year - # The first year that could be meant by YY is the current year - 50. - first = current_year - 50 - # We are seeking k such that 100k + YY is between first and first + 100. - # first <= 100k + YY < first + 100 - # 0 <= 100k + YY - first < 100 - # The value inside the comparison operators is precisely (YY - first) % 100. - # So we can calculate the century 100k as (YY - first) % 100 - (YY - first). - return first + (int(match) - first) % 100 - else: - return int(match) - else: - return current_year - -def getMonth(match_dict, current_month): - if 'M' in match_dict: - return int(match_dict['M']) - elif 'MMM' in match_dict: - return lazy_index(MONTHS, match_dict['MMM'][:3].lower()) + 1 - else: - return current_month - -def getHour(match_dict): - if 'H' in match_dict: - return int(match_dict['H']) - elif 'h' in match_dict: - hr = int(match_dict['h']) % 12 - merid = 12 if 'A' in match_dict and match_dict['A'][0] == "p" else 0 - return hr + merid - else: - return 0 - -def getMicrosecond(match_dict): - if 'S' in match_dict: - match = match_dict['S'] - return int(match + ("0"*(6-len(match))) if len(match) < 6 else match[:6]) - else: - return 0 - -# Gets the index of the first string from iter that starts with startswith -def lazy_index(l, startswith, missing=None): - for i, token in enumerate(l): - if token[:len(startswith)] == startswith: - return i - return missing diff --git a/sandbox/grist/test_moment.py b/sandbox/grist/test_moment.py index c5f21a09..724c34b6 100644 --- a/sandbox/grist/test_moment.py +++ b/sandbox/grist/test_moment.py @@ -1,7 +1,6 @@ from datetime import datetime, date, timedelta import unittest import moment -import moment_parse # Helpful strftime() format that imcludes all parts of the date including the time zone. fmt = "%Y-%m-%d %H:%M:%S %Z" @@ -60,78 +59,6 @@ class TestMoment(unittest.TestCase): [datetime(2037, 11, 1, 1, 0, 0, 0), 2140675200000, "PDT", 420, 1, 0], ] - parse_samples = [ - # Basic set - ['MM-DD-YYYY', '12-02-1999', 944092800.000000], - ['DD-MM-YYYY', '12-02-1999', 918777600.000000], - ['DD/MM/YYYY', '12/02/1999', 918777600.000000], - ['DD_MM_YYYY', '12_02_1999', 918777600.000000], - ['DD:MM:YYYY', '12:02:1999', 918777600.000000], - ['D-M-YY', '2-2-99', 917913600.000000], - ['YY', '99', 922060800.000000], - ['DD-MM-YYYY h:m:s', '12-02-1999 2:45:10', 918787510.000000], - ['DD-MM-YYYY h:m:s a', '12-02-1999 2:45:10 am', 918787510.000000], - ['DD-MM-YYYY h:m:s a', '12-02-1999 2:45:10 pm', 918830710.000000], - ['h:mm a', '12:00 pm', 1458648000.000000], - ['h:mm a', '12:30 pm', 1458649800.000000], - ['h:mm a', '12:00 am', 1458604800.000000], - ['h:mm a', '12:30 am', 1458606600.000000], - ['HH:mm', '12:00', 1458648000.000000], - ['YYYY-MM-DDTHH:mm:ss', '2011-11-11T11:11:11', 1321009871.000000], - ['ddd MMM DD HH:mm:ss YYYY', 'Tue Apr 07 22:52:51 2009', 1239144771.000000], - ['ddd MMMM DD HH:mm:ss YYYY', 'Tue April 07 22:52:51 2009', 1239144771.000000], - ['HH:mm:ss', '12:00:00', 1458648000.000000], - ['HH:mm:ss', '12:30:00', 1458649800.000000], - ['HH:mm:ss', '00:00:00', 1458604800.000000], - ['HH:mm:ss S', '00:30:00 1', 1458606600.100000], - ['HH:mm:ss SS', '00:30:00 12', 1458606600.120000], - ['HH:mm:ss SSS', '00:30:00 123', 1458606600.123000], - ['HH:mm:ss S', '00:30:00 7', 1458606600.700000], - ['HH:mm:ss SS', '00:30:00 78', 1458606600.780000], - ['HH:mm:ss SSS', '00:30:00 789', 1458606600.789000], - - # Dropped m - ['MM/DD/YYYY h:m:s a', '05/1/2012 12:25:00 p', 1335875100.000000], - ['MM/DD/YYYY h:m:s a', '05/1/2012 12:25:00 a', 1335831900.000000], - - # 2 digit year with YYYY - ['D/M/YYYY', '9/2/99', 918518400.000000], - ['D/M/YYYY', '9/2/1999', 918518400.000000], - ['D/M/YYYY', '9/2/66', -122860800.000000], - ['D/M/YYYY', '9/2/65', 3001363200.000000], - - # No separators - ['MMDDYYYY', '12021999', 944092800.000000], - ['DDMMYYYY', '12021999', 918777600.000000], - ['YYYYMMDD', '19991202', 944092800.000000], - ['DDMMMYYYY', '10Sep2001', 1000080000.000000], - - # Error forgiveness - ['MM/DD/YYYY', '12-02-1999', 944092800.000000], - ['DD/MM/YYYY', '12/02 /1999', 918777600.000000], - ['DD:MM:YYYY', '12:02 :1999', 918777600.000000], - ['D-M-YY', '2 2 99', 917913600.000000], - ['DD-MM-YYYY h:m:s', '12-02-1999 2:45:10.00', 918787510.000000], - ['h:mm a', '12:00pm', 1458648000.000000], - ['HH:mm', '1200', 1458648000.000000], - ['dddd MMMM DD HH:mm:ss YYYY', 'Tue Apr 7 22:52:51 2009', 1239144771.000000], - ['ddd MMM DD HH:mm:ss YYYY', 'Tuesday April 7 22:52:51 2009', 1239144771.000000], - ['ddd MMM Do HH:mm:ss YYYY', 'Tuesday April 7th 22:52:51 2009', 1239144771.000000] - ] - - parse_timezone_samples = [ - # Timezone corner cases - ['MM-DD-YYYY h:ma', '3-13-2016 1:59am', 'America/New_York', 1457852340], # EST - ['MM-DD-YYYY h:ma', '3-13-2016 2:00am', 'America/New_York', 1457848800], # Invalid, -1hr - ['MM-DD-YYYY h:ma', '3-13-2016 2:59am', 'America/New_York', 1457852340], # Invalid, -1hr - ['MM-DD-YYYY h:ma', '3-13-2016 3:00am', 'America/New_York', 1457852400], # EDT - ['MM-DD-YYYY h:ma', '3-13-2016 1:59am', 'America/Los_Angeles', 1457863140], # PST - ['MM-DD-YYYY h:ma', '3-13-2016 2:00am', 'America/Los_Angeles', 1457859600], # Invalid, -1hr - ['MM-DD-YYYY h:ma', '3-13-2016 2:59am', 'America/Los_Angeles', 1457863140], # Invalid, -1hr - ['MM-DD-YYYY h:ma', '3-13-2016 3:00am', 'America/Los_Angeles', 1457863200] # PDT - ] - - def assertMatches(self, data_entry, moment_obj): date, timestamp, abbr, offset, hour, minute = data_entry dt = moment_obj.datetime() @@ -183,12 +110,6 @@ class TestMoment(unittest.TestCase): self.assertEqual(dt.tzname(), abbr) self.assertEqual(dt.utcoffset(), timedelta(minutes=-offset)) - def test_parse(self): - for s in self.parse_samples: - self.assertEqual(moment_parse.parse(s[1], s[0], 'UTC', date(2016, 3, 22)), s[2]) - for s in self.parse_timezone_samples: - self.assertEqual(moment_parse.parse(s[1], s[0], s[2], date(2016, 3, 22)), s[3]) - def test_ts_to_dt(self): # Verify that ts_to_dt works as expected. value_sec = 1426291200 # 2015-03-14 00:00:00 in UTC diff --git a/sandbox/grist/test_reflist_rel.py b/sandbox/grist/test_reflist_rel.py index aa498034..1e3e7615 100644 --- a/sandbox/grist/test_reflist_rel.py +++ b/sandbox/grist/test_reflist_rel.py @@ -64,7 +64,7 @@ class TestRefListRelation(test_engine.EngineTestCase): self.apply_user_action( ['AddColumn', 'TableC', 'gristHelper_Transform', { "type": 'Ref:TableA', "isFormula": True, - "formula": "grist.Reference.typeConvert($ColB, TableA, 'ColA')", "visibleCol": 2, + "formula": "TableA.lookupOne(ColA=$ColB)", "visibleCol": 2, }]) self.apply_user_action( ['SetDisplayFormula', 'TableC', None, 7, '$gristHelper_Transform.ColA']) diff --git a/sandbox/grist/useractions.py b/sandbox/grist/useractions.py index 02702911..f015e7a2 100644 --- a/sandbox/grist/useractions.py +++ b/sandbox/grist/useractions.py @@ -1037,7 +1037,13 @@ class UserActions(object): if not clean_colinfo["isFormula"]: raise ValueError("AddColumn: cannot add a non-formula column to a summary table") - transform = col_id is not None and col_id.startswith('gristHelper_Transform') + transform = ( + col_id is not None and + col_id.startswith(( + 'gristHelper_Transform', + 'gristHelper_Converted', + )) + ) if transform: # Delete any currently existing transform columns with the same id @@ -1256,6 +1262,29 @@ class UserActions(object): finally: self._engine.out_actions.undo.append(mod_action) + @useraction + def ConvertFromColumn(self, table_id, src_col_id, dst_col_id, typ, widgetOptions, visibleColRef): + from sandbox import call_external + table = self._engine.tables[table_id] + src_col = self._docmodel.get_column_rec(table_id, src_col_id) + src_column = table.get_column(src_col_id) + row_ids = list(table.row_ids) + src_values = [encode_object(src_column.raw_get(r)) for r in row_ids] + display_values = None + if src_col.displayCol: + display_col = table.get_column(src_col.displayCol.colId) + display_values = [encode_object(display_col.raw_get(r)) for r in row_ids] + converted_values = call_external( + "convertFromColumn", + src_col.id, + typ, + widgetOptions, + visibleColRef, + src_values, + display_values, + ) + self.ModifyColumn(table_id, dst_col_id, {"type": typ}) + self.BulkUpdateRecord(table_id, row_ids, {dst_col_id: converted_values}) @useraction def CopyFromColumn(self, table_id, src_col_id, dst_col_id, widgetOptions): diff --git a/sandbox/grist/usertypes.py b/sandbox/grist/usertypes.py index 97522898..c2899cfd 100644 --- a/sandbox/grist/usertypes.py +++ b/sandbox/grist/usertypes.py @@ -138,17 +138,6 @@ class BaseColumnType(object): return objtypes.safe_repr(value_to_convert) - # This is a user-facing method, hence the camel-case naming, as for `lookupRecords` and such. - @classmethod - def typeConvert(cls, value): - """ - Convert a value from a different type to something that this type can accept, as when - explicitly converting a column type. Note that usual conversion (such as converting numbers to - strings or vice versa) will still apply to the returned value. - """ - return value - - class Text(BaseColumnType): """ Text is the type for a field holding string (text) data. @@ -180,18 +169,6 @@ class Text(BaseColumnType): def is_right_type(cls, value): return isinstance(value, (six.string_types, NoneType)) - @classmethod - def typeConvert(cls, value): - if value is None: - # When converting NULLs (that typically show up as a plain empty cell for Numeric or Date - # columns) to Text, it makes more sense to end up with a plain blank text cell. - return '' - elif isinstance(value, bool): - # Normalize True/False to true/false (Toggle columns use true/false). - return str(value).lower() - else: - return value - class Blob(BaseColumnType): """ @@ -302,13 +279,6 @@ class Date(Numeric): def is_right_type(cls, value): return isinstance(value, (float, six.integer_types, NoneType)) - @classmethod - def typeConvert(cls, value, date_format, timezone='UTC'): # pylint: disable=arguments-differ - # Note: the timezone argument is used in DateTime conversions, allows sharing this method. - try: - return moment.parse(value, date_format, timezone) - except Exception: - return value class DateTime(Date): """ @@ -370,21 +340,6 @@ class ChoiceList(BaseColumnType): return value is None or (isinstance(value, (tuple, list)) and all(isinstance(item, six.string_types) for item in value)) - @classmethod - def typeConvert(cls, value): - if value is None: - return value - if isinstance(value, six.string_types) and not value.startswith('['): - # Try to parse as CSV. If this doesn't work, we'll still try usual conversions later. - try: - tags = next(csv.reader([value])) - return tuple(t.strip() for t in tags if t.strip()) - except Exception: - pass - if not isinstance(value, (tuple, list)): - value = [Choice.typeConvert(value)] - return value - @classmethod def toString(cls, value): if isinstance(value, (tuple, list)): @@ -458,13 +413,6 @@ class Reference(Id): def typename(cls): return "Ref" - @classmethod - def typeConvert(cls, value, ref_table, visible_col=None): # pylint: disable=arguments-differ - if value and ref_table and visible_col: - return ref_table.lookupOne(**{visible_col: value}) or six.text_type(value) - else: - return value - class ReferenceList(BaseColumnType): """ @@ -500,15 +448,6 @@ class ReferenceList(BaseColumnType): return value is None or (isinstance(value, list) and all(Reference.is_right_type(val) for val in value)) - @classmethod - def typeConvert(cls, value, ref_table, visible_col=None): # noqa # pylint: disable=arguments-differ - # TODO this is based on Reference.typeConvert. - # It doesn't make much sense as a conversion but I don't know what would - if value and ref_table and visible_col: - return ref_table.lookupRecords(**{visible_col: value}) or six.text_type(value) - else: - return value - class Attachments(ReferenceList): """ @@ -516,8 +455,3 @@ class Attachments(ReferenceList): """ def __init__(self): super(Attachments, self).__init__('_grist_Attachments') - - @classmethod - def typeConvert(cls, value): # noqa # pylint: disable=arguments-differ - # Don't use ReferenceList.typeConvert which is called with a different number of arguments - return value diff --git a/test/nbrowser/gristUtils.ts b/test/nbrowser/gristUtils.ts index 3ce3a07b..fab2813a 100644 --- a/test/nbrowser/gristUtils.ts +++ b/test/nbrowser/gristUtils.ts @@ -1053,6 +1053,10 @@ export async function setType(type: RegExp, options: {skipWait?: boolean} = {}) if (!options.skipWait) { await waitForServer(); } } +export async function applyTypeTransform() { + await driver.findContent('.type_transform_prompt button', /Apply/).click(); +} + export async function isMac(): Promise { return /Darwin|Mac|iPod|iPhone|iPad/i.test((await driver.getCapabilities()).get('platform')); }