mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) New type conversion in the backend
Summary: This is https://phab.getgrist.com/D3205 plus some changes (https://github.com/dsagal/grist/compare/type-convert...type-convert-server?expand=1) that move the conversion process to the backend. A new user action ConvertFromColumn uses `call_external` so that the data engine can delegate back to ActiveDoc. Code for creating formatters and parsers is significantly refactored so that most of the logic is in `common` and can be used in different ways. Test Plan: The original diff adds plenty of tests. Reviewers: georgegevoian Reviewed By: georgegevoian Subscribers: dsagal Differential Revision: https://phab.getgrist.com/D3240
This commit is contained in:
Alex Hall
parent
4890a1fe89
commit
5d671bf0b3
@ -142,6 +142,10 @@ export class ColumnTransform extends Disposable {
|
||||
return actions.every(action => (
|
||||
// ['AddColumn', USER_TABLE, 'gristHelper_Transform', colInfo]
|
||||
(action[2] === 'gristHelper_Transform') ||
|
||||
// ['AddColumn', USER_TABLE, 'gristHelper_Converted', colInfo]
|
||||
(action[2] === 'gristHelper_Converted') ||
|
||||
// ['ConvertFromColumn', USER_TABLE, SOURCE_COLUMN, 'gristHelper_Converted']
|
||||
(action[3] === 'gristHelper_Converted') ||
|
||||
// ["SetDisplayFormula", USER_TABLE, ...]
|
||||
(action[0] === 'SetDisplayFormula') ||
|
||||
// ['UpdateRecord', '_grist_Table_column', transformColId, ...]
|
||||
@ -192,7 +196,6 @@ export class ColumnTransform extends Disposable {
|
||||
// Define variables used after await, since this will be disposed by then.
|
||||
const transformColId = this.transformColumn.colId();
|
||||
const field = this.field;
|
||||
const fieldBuilder = this._fieldBuilder;
|
||||
const origRef = this.origColumn.getRowId();
|
||||
const tableData = this._tableData;
|
||||
this.isCallPending(true);
|
||||
@ -201,17 +204,36 @@ export class ColumnTransform extends Disposable {
|
||||
// TODO: Values flicker during executing since transform column remains a formula as values are copied
|
||||
// back to the original column. The CopyFromColumn useraction really ought to be "CopyAndRemove" since
|
||||
// that seems the best way to avoid calculating the formula on wrong values.
|
||||
return await tableData.sendTableAction(['CopyFromColumn', transformColId, this.origColumn.colId(),
|
||||
JSON.stringify(fieldBuilder.options())]);
|
||||
await this.gristDoc.docData.sendActions(this.executeActions());
|
||||
}
|
||||
} finally {
|
||||
// Wait until the change completed to set column back, to avoid value flickering.
|
||||
field.colRef(origRef);
|
||||
void tableData.sendTableAction(['RemoveColumn', transformColId]);
|
||||
this.cleanup();
|
||||
this.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The user actions to send when actually executing the transform.
|
||||
*/
|
||||
protected executeActions(): UserAction[] {
|
||||
return [
|
||||
[
|
||||
'CopyFromColumn',
|
||||
this._tableData.tableId,
|
||||
this.transformColumn.colId(),
|
||||
this.origColumn.colId(),
|
||||
JSON.stringify(this._fieldBuilder.options()),
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
protected cleanup() {
|
||||
// For overriding
|
||||
}
|
||||
|
||||
protected getIdentityFormula() {
|
||||
return 'return $' + this.origColumn.colId();
|
||||
}
|
||||
|
||||
@ -6,7 +6,6 @@
|
||||
|
||||
import {DocModel} from 'app/client/models/DocModel';
|
||||
import {ColumnRec} from 'app/client/models/entities/ColumnRec';
|
||||
import * as UserType from 'app/client/widgets/UserType';
|
||||
import * as gristTypes from 'app/common/gristTypes';
|
||||
import {isFullReferencingType} from 'app/common/gristTypes';
|
||||
import * as gutil from 'app/common/gutil';
|
||||
@ -90,7 +89,7 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
|
||||
type: addColTypeSuffix(toTypeMaybeFull, origCol, docModel),
|
||||
isFormula: true,
|
||||
visibleCol: 0,
|
||||
formula: "", // Will be filled in at the end.
|
||||
formula: "CURRENT_CONVERSION(rec)",
|
||||
};
|
||||
|
||||
const prevOptions = origCol.widgetOptionsJson.peek() || {};
|
||||
@ -139,23 +138,35 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
|
||||
case 'RefList':
|
||||
{
|
||||
// Set suggested destination table and visible column.
|
||||
// Null if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen).
|
||||
const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`)!;
|
||||
// Undefined if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen).
|
||||
const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`) || undefined;
|
||||
|
||||
// Finds a reference suggestion column and sets it as the current reference value.
|
||||
const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100);
|
||||
if (!columnData) { break; }
|
||||
columnData.delete(gristTypes.getDefaultForType(origCol.type()));
|
||||
let suggestedColRef: number;
|
||||
let suggestedTableId: string;
|
||||
const origColTypeInfo = gristTypes.extractInfoFromColType(origCol.type.peek());
|
||||
if (!optTableId && origColTypeInfo.type === "Ref" || origColTypeInfo.type === "RefList") {
|
||||
// When converting between Ref and Reflist, initially suggest the same table and visible column.
|
||||
// When converting, if the table is the same, it's a special case.
|
||||
// The visible column will not affect conversion.
|
||||
// It will simply wrap the reference (row ID) in a list or extract the one element of a reference list.
|
||||
suggestedColRef = origCol.visibleCol.peek();
|
||||
suggestedTableId = origColTypeInfo.tableId;
|
||||
} else {
|
||||
// Finds a reference suggestion column and sets it as the current reference value.
|
||||
const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100);
|
||||
if (!columnData) { break; }
|
||||
columnData.delete(gristTypes.getDefaultForType(origCol.type()));
|
||||
|
||||
// 'findColFromValues' function requires an array since it sends the values to the sandbox.
|
||||
const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId);
|
||||
const suggestedColRef = matches.find(match => match !== origCol.getRowId());
|
||||
if (!suggestedColRef) { break; }
|
||||
const suggestedCol = docModel.columns.getRowModel(suggestedColRef);
|
||||
const suggestedTableId = suggestedCol.table().tableId();
|
||||
if (optTableId && suggestedTableId !== optTableId) {
|
||||
console.warn("Inappropriate column received from findColFromValues");
|
||||
break;
|
||||
// 'findColFromValues' function requires an array since it sends the values to the sandbox.
|
||||
const matches: number[] = await docModel.docData.findColFromValues(Array.from(columnData), 2, optTableId);
|
||||
suggestedColRef = matches.find(match => match !== origCol.getRowId())!;
|
||||
if (!suggestedColRef) { break; }
|
||||
const suggestedCol = docModel.columns.getRowModel(suggestedColRef);
|
||||
suggestedTableId = suggestedCol.table().tableId();
|
||||
if (optTableId && suggestedTableId !== optTableId) {
|
||||
console.warn("Inappropriate column received from findColFromValues");
|
||||
break;
|
||||
}
|
||||
}
|
||||
colInfo.type = `${toType}:${suggestedTableId}`;
|
||||
colInfo.visibleCol = suggestedColRef;
|
||||
@ -163,11 +174,9 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
|
||||
}
|
||||
}
|
||||
|
||||
const newOptions = UserType.mergeOptions(widgetOptions || {}, colInfo.type);
|
||||
if (widgetOptions) {
|
||||
colInfo.widgetOptions = JSON.stringify(widgetOptions);
|
||||
}
|
||||
colInfo.formula = getDefaultFormula(docModel, origCol, colInfo.type, colInfo.visibleCol, newOptions);
|
||||
return colInfo;
|
||||
}
|
||||
|
||||
@ -184,62 +193,6 @@ export async function setDisplayFormula(
|
||||
}
|
||||
}
|
||||
|
||||
// Given the original column and info about the new column, returns the formula to use for the
|
||||
// transform column to do the transformation.
|
||||
export function getDefaultFormula(
|
||||
docModel: DocModel, origCol: ColumnRec, newType: string,
|
||||
newVisibleCol: number, newWidgetOptions: any): string {
|
||||
|
||||
const colId = origCol.colId();
|
||||
const oldVisibleColName = isReferenceCol(origCol) ?
|
||||
getVisibleColName(docModel, origCol.visibleCol()) : undefined;
|
||||
|
||||
let origValFormula = oldVisibleColName ?
|
||||
// The `str()` below converts AltText to plain text.
|
||||
`($${colId}.${oldVisibleColName}
|
||||
if ISREF($${colId}) or ISREFLIST($${colId})
|
||||
else str($${colId}))`
|
||||
: `$${colId}`;
|
||||
|
||||
if (origCol.type.peek() === 'ChoiceList') {
|
||||
origValFormula = `grist.ChoiceList.toString($${colId})`;
|
||||
}
|
||||
|
||||
const toTypePure: string = gristTypes.extractTypeFromColType(newType);
|
||||
|
||||
// The args are used to construct the call to grist.TYPE.typeConvert(value, [params]).
|
||||
// Optional parameters depend on the type; see sandbox/grist/usertypes.py
|
||||
const args: string[] = [origValFormula];
|
||||
switch (toTypePure) {
|
||||
case 'Ref':
|
||||
case 'RefList':
|
||||
{
|
||||
const table = gutil.removePrefix(newType, toTypePure + ":");
|
||||
args.push(table || 'None');
|
||||
const visibleColName = getVisibleColName(docModel, newVisibleCol);
|
||||
if (visibleColName) {
|
||||
args.push(q(visibleColName));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'Date': {
|
||||
args.push(q(newWidgetOptions.dateFormat));
|
||||
break;
|
||||
}
|
||||
case 'DateTime': {
|
||||
const timezone = gutil.removePrefix(newType, "DateTime:") || '';
|
||||
const format = newWidgetOptions.dateFormat + ' ' + newWidgetOptions.timeFormat;
|
||||
args.push(q(format), q(timezone));
|
||||
break;
|
||||
}
|
||||
}
|
||||
return `grist.${gristTypes.getGristType(toTypePure)}.typeConvert(${args.join(', ')})`;
|
||||
}
|
||||
|
||||
function q(value: string): string {
|
||||
return "'" + value.replace(/'/g, "\\'") + "'";
|
||||
}
|
||||
|
||||
// Returns the name of the visibleCol given its rowId.
|
||||
function getVisibleColName(docModel: DocModel, visibleColRef: number): string|undefined {
|
||||
return visibleColRef ? docModel.columns.getRowModel(visibleColRef).colId() : undefined;
|
||||
|
||||
@ -10,13 +10,12 @@ import {ColumnTransform} from 'app/client/components/ColumnTransform';
|
||||
import {GristDoc} from 'app/client/components/GristDoc';
|
||||
import * as TypeConversion from 'app/client/components/TypeConversion';
|
||||
import {reportError} from 'app/client/models/errors';
|
||||
import * as modelUtil from 'app/client/models/modelUtil';
|
||||
import {cssButtonRow} from 'app/client/ui/RightPanel';
|
||||
import {basicButton, primaryButton} from 'app/client/ui2018/buttons';
|
||||
import {testId} from 'app/client/ui2018/cssVars';
|
||||
import {FieldBuilder} from 'app/client/widgets/FieldBuilder';
|
||||
import {NewAbstractWidget} from 'app/client/widgets/NewAbstractWidget';
|
||||
import {ColValues} from 'app/common/DocActions';
|
||||
import {ColValues, UserAction} from 'app/common/DocActions';
|
||||
import {Computed, dom, fromKo, Observable} from 'grainjs';
|
||||
import isEmpty = require('lodash/isEmpty');
|
||||
import pickBy = require('lodash/pickBy');
|
||||
@ -86,15 +85,6 @@ export class TypeTransform extends ColumnTransform {
|
||||
);
|
||||
}
|
||||
|
||||
protected async resetToDefaultFormula() {
|
||||
if (!this.isFinalizing()) {
|
||||
const toType = this.transformColumn.type.peek();
|
||||
const formula = TypeConversion.getDefaultFormula(this.gristDoc.docModel, this.origColumn,
|
||||
toType, this.field.visibleColRef(), this.field.widgetOptionsJson());
|
||||
await modelUtil.setSaveValue(this.transformColumn.formula, formula);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Overrides parent method to initialize the transform column with guesses as to the particular
|
||||
* type and column options.
|
||||
@ -103,20 +93,55 @@ export class TypeTransform extends ColumnTransform {
|
||||
protected async addTransformColumn(toType: string) {
|
||||
const docModel = this.gristDoc.docModel;
|
||||
const colInfo = await TypeConversion.prepTransformColInfo(docModel, this.origColumn, this.origDisplayCol, toType);
|
||||
const newColInfo = await this._tableData.sendTableAction(['AddColumn', 'gristHelper_Transform', colInfo]);
|
||||
const tcol = docModel.columns.getRowModel(newColInfo.colRef);
|
||||
await TypeConversion.setDisplayFormula(docModel, tcol);
|
||||
return newColInfo.colRef;
|
||||
const newColInfos = await this._tableData.sendTableActions([
|
||||
['AddColumn', 'gristHelper_Converted', {...colInfo, isFormula: false, formula: ''}],
|
||||
['AddColumn', 'gristHelper_Transform', colInfo],
|
||||
]);
|
||||
const transformColRef = newColInfos[1].colRef;
|
||||
this.transformColumn = docModel.columns.getRowModel(transformColRef);
|
||||
await this.convertValues();
|
||||
return transformColRef;
|
||||
}
|
||||
|
||||
protected convertValuesActions(): UserAction[] {
|
||||
const tableId = this._tableData.tableId;
|
||||
const srcColId = this.origColumn.colId.peek();
|
||||
const dstColId = "gristHelper_Converted";
|
||||
const type = this.transformColumn.type.peek();
|
||||
const widgetOptions = this.transformColumn.widgetOptions.peek();
|
||||
const visibleColRef = this.transformColumn.visibleCol.peek();
|
||||
return [['ConvertFromColumn', tableId, srcColId, dstColId, type, widgetOptions, visibleColRef]];
|
||||
}
|
||||
|
||||
protected async convertValues() {
|
||||
await Promise.all([
|
||||
this.gristDoc.docData.sendActions(this.convertValuesActions()),
|
||||
TypeConversion.setDisplayFormula(this.gristDoc.docModel, this.transformColumn),
|
||||
]);
|
||||
}
|
||||
|
||||
protected executeActions(): UserAction[] {
|
||||
return [...this.convertValuesActions(), ...super.executeActions()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Overrides parent method to subscribe to changes to the transform column.
|
||||
*/
|
||||
protected postAddTransformColumn() {
|
||||
// When a user-initiated change is saved to type or widgetOptions, update the formula.
|
||||
this.autoDispose(this.transformColumn.type.subscribe(this.resetToDefaultFormula, this, "save"));
|
||||
this.autoDispose(this.transformColumn.visibleCol.subscribe(this.resetToDefaultFormula, this, "save"));
|
||||
this.autoDispose(this.field.widgetOptionsJson.subscribe(this.resetToDefaultFormula, this, "save"));
|
||||
// When a user-initiated change is saved to type or widgetOptions, reconvert the values
|
||||
// Need to subscribe to both 'change' and 'save' for type which can come from setting the type itself
|
||||
// or e.g. a change to DateTime timezone.
|
||||
this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "change"));
|
||||
this.autoDispose(this.transformColumn.type.subscribe(this.convertValues, this, "save"));
|
||||
this.autoDispose(this.transformColumn.visibleCol.subscribe(this.convertValues, this, "save"));
|
||||
this.autoDispose(this.field.widgetOptionsJson.subscribe(this.convertValues, this, "save"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Overrides parent method to delete extra column
|
||||
*/
|
||||
protected cleanup() {
|
||||
void this._tableData.sendTableAction(['RemoveColumn', 'gristHelper_Converted']);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -129,9 +154,10 @@ export class TypeTransform extends ColumnTransform {
|
||||
const tcol = this.transformColumn;
|
||||
const changedInfo = pickBy(colInfo, (val, key) =>
|
||||
(val !== tcol[key as keyof TypeConversion.ColInfo].peek()));
|
||||
return Promise.all([
|
||||
isEmpty(changedInfo) ? undefined : tcol.updateColValues(changedInfo as ColValues),
|
||||
TypeConversion.setDisplayFormula(docModel, tcol, changedInfo.visibleCol)
|
||||
]);
|
||||
if (!isEmpty(changedInfo)) {
|
||||
// Update the transform column, particularly the type.
|
||||
// This will trigger the subscription in postAddTransformColumn and lead to calling convertValues.
|
||||
await tcol.updateColValues(changedInfo as ColValues);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -22,7 +22,7 @@ import {urlState} from 'app/client/models/gristUrlState';
|
||||
import * as MetaRowModel from 'app/client/models/MetaRowModel';
|
||||
import * as MetaTableModel from 'app/client/models/MetaTableModel';
|
||||
import * as rowset from 'app/client/models/rowset';
|
||||
import {isHiddenTable} from 'app/client/models/isHiddenTable';
|
||||
import {isHiddenTable} from 'app/common/isHiddenTable';
|
||||
import {schema, SchemaTypes} from 'app/common/schema';
|
||||
|
||||
import {ACLRuleRec, createACLRuleRec} from 'app/client/models/entities/ACLRuleRec';
|
||||
|
||||
@ -2,8 +2,13 @@ import {KoArray} from 'app/client/lib/koArray';
|
||||
import {DocModel, IRowModel, recordSet, refRecord, TableRec, ViewFieldRec} from 'app/client/models/DocModel';
|
||||
import {jsonObservable, ObjObservable} from 'app/client/models/modelUtil';
|
||||
import * as gristTypes from 'app/common/gristTypes';
|
||||
import {getReferencedTableId, isFullReferencingType} from 'app/common/gristTypes';
|
||||
import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter';
|
||||
import {getReferencedTableId} from 'app/common/gristTypes';
|
||||
import {
|
||||
BaseFormatter,
|
||||
createFullFormatterRaw,
|
||||
createVisibleColFormatterRaw,
|
||||
FullFormatterArgs
|
||||
} from 'app/common/ValueFormatter';
|
||||
import * as ko from 'knockout';
|
||||
|
||||
// Represents a column in a user-defined table.
|
||||
@ -124,38 +129,23 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void {
|
||||
|
||||
// Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol
|
||||
// associated with this column. If no visible column available, return formatting for the column itself.
|
||||
this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this, docModel));
|
||||
this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'vcol'));
|
||||
|
||||
this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, this.visibleColFormatter()));
|
||||
}
|
||||
|
||||
export function visibleColFormatterForRec(
|
||||
rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel
|
||||
): BaseFormatter {
|
||||
const vcol = rec.visibleColModel();
|
||||
const documentSettings = docModel.docInfoRow.documentSettingsJson();
|
||||
const type = colRec.type();
|
||||
if (isFullReferencingType(type)) {
|
||||
if (vcol.getRowId() === 0) {
|
||||
// This column displays the Row ID, e.g. Table1[2]
|
||||
// referencedTableId may actually be empty if the table is hidden
|
||||
const referencedTableId: string = colRec.refTable()?.tableId() || "";
|
||||
return createFormatter('Id', {tableId: referencedTableId}, documentSettings);
|
||||
} else {
|
||||
return createFormatter(vcol.type(), vcol.widgetOptionsJson(), documentSettings);
|
||||
}
|
||||
} else {
|
||||
// For non-reference columns, there's no 'visible column' and we just return a regular formatter
|
||||
return createFormatter(type, rec.widgetOptionsJson(), documentSettings);
|
||||
}
|
||||
this.formatter = ko.pureComputed(() => formatterForRec(this, this, docModel, 'full'));
|
||||
}
|
||||
|
||||
export function formatterForRec(
|
||||
rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, visibleColFormatter: BaseFormatter
|
||||
rec: ColumnRec | ViewFieldRec, colRec: ColumnRec, docModel: DocModel, kind: 'full' | 'vcol'
|
||||
): BaseFormatter {
|
||||
const type = colRec.type();
|
||||
// Ref/RefList columns delegate most formatting to the visibleColFormatter
|
||||
const widgetOpts = {...rec.widgetOptionsJson(), visibleColFormatter};
|
||||
const documentSettings = docModel.docInfoRow.documentSettingsJson();
|
||||
return createFormatter(type, widgetOpts, documentSettings);
|
||||
const vcol = rec.visibleColModel();
|
||||
const func = kind === 'full' ? createFullFormatterRaw : createVisibleColFormatterRaw;
|
||||
const args: FullFormatterArgs = {
|
||||
docData: docModel.docData,
|
||||
type: colRec.type(),
|
||||
widgetOpts: rec.widgetOptionsJson(),
|
||||
visibleColType: vcol?.type(),
|
||||
visibleColWidgetOpts: vcol?.widgetOptionsJson(),
|
||||
docSettings: docModel.docInfoRow.documentSettingsJson(),
|
||||
};
|
||||
return func(args);
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel';
|
||||
import {formatterForRec, visibleColFormatterForRec} from 'app/client/models/entities/ColumnRec';
|
||||
import {formatterForRec} from 'app/client/models/entities/ColumnRec';
|
||||
import * as modelUtil from 'app/client/models/modelUtil';
|
||||
import * as UserType from 'app/client/widgets/UserType';
|
||||
import {DocumentSettings} from 'app/common/DocumentSettings';
|
||||
@ -172,13 +172,14 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void
|
||||
|
||||
// Helper for Reference/ReferenceList columns, which returns a formatter according to the visibleCol
|
||||
// associated with this field. If no visible column available, return formatting for the field itself.
|
||||
this.visibleColFormatter = ko.pureComputed(() => visibleColFormatterForRec(this, this.column(), docModel));
|
||||
this.visibleColFormatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'vcol'));
|
||||
|
||||
this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, this.visibleColFormatter()));
|
||||
this.formatter = ko.pureComputed(() => formatterForRec(this, this.column(), docModel, 'full'));
|
||||
|
||||
this.createValueParser = function() {
|
||||
const fieldRef = this.useColOptions.peek() ? undefined : this.id.peek();
|
||||
return createParser(docModel.docData, this.colRef.peek(), fieldRef);
|
||||
const parser = createParser(docModel.docData, this.colRef.peek(), fieldRef);
|
||||
return parser.cleanParse.bind(parser);
|
||||
};
|
||||
|
||||
// The widgetOptions to read and write: either the column's or the field's own.
|
||||
|
||||
@ -3,7 +3,7 @@ import { duplicatePage } from "app/client/components/duplicatePage";
|
||||
import { GristDoc } from "app/client/components/GristDoc";
|
||||
import { PageRec } from "app/client/models/DocModel";
|
||||
import { urlState } from "app/client/models/gristUrlState";
|
||||
import { isHiddenTable } from 'app/client/models/isHiddenTable';
|
||||
import { isHiddenTable } from 'app/common/isHiddenTable';
|
||||
import * as MetaTableModel from "app/client/models/MetaTableModel";
|
||||
import { find as findInTree, fromTableData, TreeItemRecord, TreeRecord,
|
||||
TreeTableData} from "app/client/models/TreeModel";
|
||||
|
||||
255
app/common/ValueConverter.ts
Normal file
255
app/common/ValueConverter.ts
Normal file
@ -0,0 +1,255 @@
|
||||
import {DocData} from 'app/common/DocData';
|
||||
import * as gristTypes from 'app/common/gristTypes';
|
||||
import {isList} from 'app/common/gristTypes';
|
||||
import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFormatter';
|
||||
import {
|
||||
createParserOrFormatterArgumentsRaw,
|
||||
createParserRaw,
|
||||
ReferenceListParser,
|
||||
ReferenceParser,
|
||||
ValueParser
|
||||
} from 'app/common/ValueParser';
|
||||
import {CellValue, GristObjCode} from 'app/plugin/GristData';
|
||||
|
||||
|
||||
/**
|
||||
* Base class for converting values from one type to another with the convert() method.
|
||||
* Has a formatter for the source column
|
||||
* and a parser for the destination column.
|
||||
*
|
||||
* The default convert() is for non-list destination types, so if the source value
|
||||
* is a list it only converts nicely if the list contains exactly one element.
|
||||
*/
|
||||
export class ValueConverter {
|
||||
constructor(public formatter: BaseFormatter, public parser: ValueParser) {
|
||||
}
|
||||
|
||||
public convert(value: any): any {
|
||||
if (isList(value)) {
|
||||
if (value.length === 1) {
|
||||
// Empty list: ['L']
|
||||
return null;
|
||||
} else if (value.length === 2) {
|
||||
// Singleton list: ['L', value]
|
||||
// Convert just that one value.
|
||||
value = value[1];
|
||||
} else {
|
||||
// List with multiple values. Since we're converting to just one value,
|
||||
// format the whole thing as text, which is an error for most types.
|
||||
return this.formatter.formatAny(value);
|
||||
}
|
||||
}
|
||||
return this.convertInner(value);
|
||||
}
|
||||
|
||||
protected convertInner(value: any): any {
|
||||
const formatted = this.formatter.formatAny(value);
|
||||
return this.parser.cleanParse(formatted);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Base class for converting to a list type (Reference List or Choice List).
|
||||
*
|
||||
* Wraps single values in a list, and converts lists elementwise.
|
||||
*/
|
||||
class ListConverter extends ValueConverter {
|
||||
// Don't parse strings like "Smith, John" which may look like lists but represent a single choice.
|
||||
// TODO this works when the source is a Choice column, but not when it's a Reference to a Choice column.
|
||||
// But the guessed choices are also broken in that case.
|
||||
private _choices: Set<string> = new Set((this.formatter.widgetOpts as any).choices || []);
|
||||
|
||||
public convert(value: any): any {
|
||||
if (typeof value === "string" && !this._choices.has(value)) {
|
||||
// Parse CSV/JSON
|
||||
return this.parser.cleanParse(value);
|
||||
}
|
||||
const values = isList(value) ? value.slice(1) : [value];
|
||||
if (!values.length || value == null) {
|
||||
return null;
|
||||
}
|
||||
return this.handleValues(value, values.map(v => this.convertInner(v)));
|
||||
}
|
||||
|
||||
protected handleValues(originalValue: any, values: any[]) {
|
||||
return ['L', ...values];
|
||||
}
|
||||
}
|
||||
|
||||
class ChoiceListConverter extends ListConverter {
|
||||
/**
|
||||
* Convert each source value to a 'Choice'
|
||||
*/
|
||||
protected convertInner(value: any): any {
|
||||
return this.formatter.formatAny(value);
|
||||
}
|
||||
}
|
||||
|
||||
class ReferenceListConverter extends ListConverter {
|
||||
private _innerConverter = new ReferenceConverter(
|
||||
this.formatter,
|
||||
new ReferenceParser("Ref", this.parser.widgetOpts, this.parser.docSettings),
|
||||
);
|
||||
|
||||
constructor(public formatter: BaseFormatter, public parser: ReferenceListParser) {
|
||||
super(formatter, parser);
|
||||
// Prevent the parser from looking up reference values in the frontend.
|
||||
// Leave it to the data engine which has a much more efficient algorithm for long lists of values.
|
||||
delete parser.tableData;
|
||||
}
|
||||
|
||||
public handleValues(originalValue: any, values: any[]): any {
|
||||
const result = [];
|
||||
let lookupColumn: string = "";
|
||||
const raw = this.formatter.formatAny(originalValue); // AltText if the reference lookup fails
|
||||
for (const value of values) {
|
||||
if (typeof value === "string") {
|
||||
// Failed to parse one of the references, so return a raw string for the whole thing
|
||||
return raw;
|
||||
} else {
|
||||
// value is a lookup tuple: ['l', value, options]
|
||||
result.push(value[1]);
|
||||
lookupColumn = value[2].column;
|
||||
}
|
||||
}
|
||||
return ['l', result, {column: lookupColumn, raw}];
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert each source value to a 'Reference'
|
||||
*/
|
||||
protected convertInner(value: any): any {
|
||||
return this._innerConverter.convert(value);
|
||||
}
|
||||
}
|
||||
|
||||
class ReferenceConverter extends ValueConverter {
|
||||
private _innerConverter: ValueConverter = createConverter(this.formatter, this.parser.visibleColParser);
|
||||
|
||||
constructor(public formatter: BaseFormatter, public parser: ReferenceParser) {
|
||||
super(formatter, parser);
|
||||
// Prevent the parser from looking up reference values in the frontend.
|
||||
// Leave it to the data engine which has a much more efficient algorithm for long lists of values.
|
||||
delete parser.tableData;
|
||||
}
|
||||
|
||||
protected convertInner(value: any): any {
|
||||
// Convert to the type of the visible column.
|
||||
const converted = this._innerConverter.convert(value);
|
||||
return this.parser.lookup(converted, this.formatter.formatAny(value));
|
||||
}
|
||||
}
|
||||
|
||||
class NumericConverter extends ValueConverter {
|
||||
protected convertInner(value: any): any {
|
||||
if (typeof value === "boolean") {
|
||||
return value ? 1 : 0;
|
||||
}
|
||||
return super.convertInner(value);
|
||||
}
|
||||
}
|
||||
|
||||
class DateConverter extends ValueConverter {
|
||||
private _sourceType = gristTypes.extractInfoFromColType(this.formatter.type);
|
||||
|
||||
protected convertInner(value: any): any {
|
||||
// When converting Date->DateTime, DateTime->Date, or between DateTime timezones,
|
||||
// it's important to send an encoded Date/DateTime object rather than just a timestamp number
|
||||
// so that the data engine knows what to do in do_convert, especially regarding timezones.
|
||||
// If the source column is a Reference to a Date/DateTime then `value` is already
|
||||
// an encoded object from the display column which has type Any.
|
||||
value = gristTypes.reencodeAsAny(value, this._sourceType);
|
||||
if (Array.isArray(value) && (
|
||||
value[0] === GristObjCode.Date ||
|
||||
value[0] === GristObjCode.DateTime
|
||||
)) {
|
||||
return value;
|
||||
}
|
||||
return super.convertInner(value);
|
||||
}
|
||||
}
|
||||
|
||||
export const valueConverterClasses: { [type: string]: typeof ValueConverter } = {
|
||||
Date: DateConverter,
|
||||
DateTime: DateConverter,
|
||||
ChoiceList: ChoiceListConverter,
|
||||
Ref: ReferenceConverter,
|
||||
RefList: ReferenceListConverter,
|
||||
Numeric: NumericConverter,
|
||||
Int: NumericConverter,
|
||||
};
|
||||
|
||||
export function createConverter(formatter: BaseFormatter, parser: ValueParser) {
|
||||
const cls = valueConverterClasses[gristTypes.extractTypeFromColType(parser.type)] || ValueConverter;
|
||||
return new cls(formatter, parser);
|
||||
}
|
||||
|
||||
/**
|
||||
* Used by the ConvertFromColumn user action in the data engine.
|
||||
* The higher order function separates docData (passed by ActiveDoc)
|
||||
* from the arguments passed to call_external in Python.
|
||||
*/
|
||||
export function convertFromColumn(docData: DocData) {
|
||||
return function(
|
||||
sourceColRef: number,
|
||||
type: string,
|
||||
widgetOpts: string,
|
||||
visibleColRef: number,
|
||||
values: ReadonlyArray<CellValue>,
|
||||
displayColValues?: ReadonlyArray<CellValue>,
|
||||
): CellValue[] {
|
||||
const formatter = createFullFormatterFromDocData(docData, sourceColRef);
|
||||
const parser = createParserRaw(
|
||||
...createParserOrFormatterArgumentsRaw(docData, type, widgetOpts, visibleColRef)
|
||||
);
|
||||
const converter = createConverter(formatter, parser);
|
||||
return convertValues(converter, values, displayColValues || values);
|
||||
};
|
||||
}
|
||||
|
||||
export function convertValues(
|
||||
converter: ValueConverter,
|
||||
// Raw values from the actual column, e.g. row IDs for reference columns
|
||||
values: ReadonlyArray<CellValue>,
|
||||
// Values from the display column, which is the same as the raw values for non-referencing columns.
|
||||
// In almost all cases these are the values that actually matter and get converted.
|
||||
displayColValues: ReadonlyArray<CellValue>,
|
||||
): CellValue[] {
|
||||
// Converting Ref <-> RefList without changing the target table is a special case - see prepTransformColInfo.
|
||||
// In this case we deal with the actual row IDs stored in the real column,
|
||||
// whereas in all other cases we use display column values.
|
||||
const sourceType = gristTypes.extractInfoFromColType(converter.formatter.type);
|
||||
const targetType = gristTypes.extractInfoFromColType(converter.parser.type);
|
||||
const refToRefList = (
|
||||
sourceType.type === "Ref" &&
|
||||
targetType.type === "RefList" &&
|
||||
sourceType.tableId === targetType.tableId
|
||||
);
|
||||
const refListToRef = (
|
||||
sourceType.type === "RefList" &&
|
||||
targetType.type === "Ref" &&
|
||||
sourceType.tableId === targetType.tableId
|
||||
);
|
||||
|
||||
return displayColValues.map((displayVal, i) => {
|
||||
const actualValue = values[i];
|
||||
|
||||
if (refToRefList && typeof actualValue === "number") {
|
||||
if (actualValue === 0) {
|
||||
return null;
|
||||
} else {
|
||||
return ["L", actualValue];
|
||||
}
|
||||
} else if (refListToRef && isList(actualValue)) {
|
||||
if (actualValue.length === 1) {
|
||||
// Empty list: ['L']
|
||||
return 0;
|
||||
} else if (actualValue.length === 2) {
|
||||
// Singleton list: ['L', rowId]
|
||||
return actualValue[1];
|
||||
}
|
||||
}
|
||||
|
||||
return converter.convert(displayVal);
|
||||
});
|
||||
}
|
||||
@ -2,11 +2,14 @@
|
||||
|
||||
import {csvEncodeRow} from 'app/common/csvFormat';
|
||||
import {CellValue} from 'app/common/DocActions';
|
||||
import {DocData} from 'app/common/DocData';
|
||||
import {DocumentSettings} from 'app/common/DocumentSettings';
|
||||
import {getReferencedTableId, isList} from 'app/common/gristTypes';
|
||||
import * as gristTypes from 'app/common/gristTypes';
|
||||
import {getReferencedTableId, isList} from 'app/common/gristTypes';
|
||||
import * as gutil from 'app/common/gutil';
|
||||
import {isHiddenTable} from 'app/common/isHiddenTable';
|
||||
import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
|
||||
import {createParserOrFormatterArguments, ReferenceParsingOptions} from 'app/common/ValueParser';
|
||||
import {GristObjCode} from 'app/plugin/GristData';
|
||||
import {decodeObject, GristDateTime} from 'app/plugin/objtypes';
|
||||
import * as moment from 'moment-timezone';
|
||||
@ -280,3 +283,70 @@ export function createFormatter(type: string, widgetOpts: FormatOptions, docSett
|
||||
const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter;
|
||||
return new ctor(type, widgetOpts, docSettings);
|
||||
}
|
||||
|
||||
export interface FullFormatterArgs {
|
||||
docData: DocData;
|
||||
type: string;
|
||||
widgetOpts: FormatOptions;
|
||||
visibleColType: string;
|
||||
visibleColWidgetOpts: FormatOptions;
|
||||
docSettings: DocumentSettings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a constructor
|
||||
* with a format function that can properly convert a value passed to it into the
|
||||
* right format for that column.
|
||||
*
|
||||
* Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
|
||||
* instead of the table column.
|
||||
*/
|
||||
export function createFullFormatterFromDocData(
|
||||
docData: DocData,
|
||||
colRef: number,
|
||||
fieldRef?: number,
|
||||
): BaseFormatter {
|
||||
const [type, widgetOpts, docSettings] = createParserOrFormatterArguments(docData, colRef, fieldRef);
|
||||
const {visibleColType, visibleColWidgetOpts} = widgetOpts as ReferenceParsingOptions;
|
||||
return createFullFormatterRaw({
|
||||
docData,
|
||||
type,
|
||||
widgetOpts,
|
||||
visibleColType,
|
||||
visibleColWidgetOpts,
|
||||
docSettings,
|
||||
});
|
||||
}
|
||||
|
||||
export function createFullFormatterRaw(args: FullFormatterArgs) {
|
||||
const {type, widgetOpts, docSettings} = args;
|
||||
const visibleColFormatter = createVisibleColFormatterRaw(args);
|
||||
return createFormatter(type, {...widgetOpts, visibleColFormatter}, docSettings);
|
||||
}
|
||||
|
||||
export function createVisibleColFormatterRaw(
|
||||
{
|
||||
docData,
|
||||
docSettings,
|
||||
type,
|
||||
visibleColType,
|
||||
visibleColWidgetOpts,
|
||||
widgetOpts
|
||||
}: FullFormatterArgs
|
||||
): BaseFormatter {
|
||||
let referencedTableId = gristTypes.getReferencedTableId(type);
|
||||
if (!referencedTableId) {
|
||||
return createFormatter(type, widgetOpts, docSettings);
|
||||
} else if (visibleColType) {
|
||||
return createFormatter(visibleColType, visibleColWidgetOpts, docSettings);
|
||||
} else {
|
||||
// This column displays the Row ID, e.g. Table1[2]
|
||||
// Make referencedTableId empty if the table is hidden
|
||||
const tablesData = docData.getMetaTable("_grist_Tables");
|
||||
const tableRef = tablesData.findRow("tableId", referencedTableId);
|
||||
if (isHiddenTable(tablesData, tableRef)) {
|
||||
referencedTableId = "";
|
||||
}
|
||||
return createFormatter('Id', {tableId: referencedTableId}, docSettings);
|
||||
}
|
||||
}
|
||||
|
||||
@ -33,6 +33,9 @@ export class ValueParser {
|
||||
|
||||
}
|
||||
|
||||
class IdentityParser extends ValueParser {
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as basic Value parser, but will return null if a value is an empty string.
|
||||
*/
|
||||
@ -117,7 +120,7 @@ class ChoiceListParser extends ValueParser {
|
||||
* stored on the field. These have to be specially derived
|
||||
* for referencing columns. See createParser.
|
||||
*/
|
||||
interface ReferenceParsingOptions {
|
||||
export interface ReferenceParsingOptions {
|
||||
visibleColId: string;
|
||||
visibleColType: string;
|
||||
visibleColWidgetOpts: FormatOptions;
|
||||
@ -129,18 +132,22 @@ interface ReferenceParsingOptions {
|
||||
|
||||
export class ReferenceParser extends ValueParser {
|
||||
public widgetOpts: ReferenceParsingOptions;
|
||||
|
||||
protected _visibleColId = this.widgetOpts.visibleColId;
|
||||
protected _tableData = this.widgetOpts.tableData;
|
||||
protected _visibleColParser = createParserRaw(
|
||||
public tableData = this.widgetOpts.tableData;
|
||||
public visibleColParser = createParserRaw(
|
||||
this.widgetOpts.visibleColType,
|
||||
this.widgetOpts.visibleColWidgetOpts,
|
||||
this.docSettings,
|
||||
);
|
||||
|
||||
protected _visibleColId = this.widgetOpts.visibleColId;
|
||||
|
||||
public parse(raw: string): any {
|
||||
let value = this._visibleColParser(raw);
|
||||
if (!value || !raw) {
|
||||
const value = this.visibleColParser.cleanParse(raw);
|
||||
return this.lookup(value, raw);
|
||||
}
|
||||
|
||||
public lookup(value: any, raw: string): any {
|
||||
if (value == null || value === "" || !raw) {
|
||||
return 0; // default value for a reference column
|
||||
}
|
||||
|
||||
@ -154,7 +161,7 @@ export class ReferenceParser extends ValueParser {
|
||||
}
|
||||
}
|
||||
|
||||
if (!this._tableData?.isLoaded) {
|
||||
if (!this.tableData?.isLoaded) {
|
||||
const options: { column: string, raw?: string } = {column: this._visibleColId};
|
||||
if (value !== raw) {
|
||||
options.raw = raw;
|
||||
@ -162,7 +169,7 @@ export class ReferenceParser extends ValueParser {
|
||||
return ['l', value, options];
|
||||
}
|
||||
|
||||
return this._tableData.findMatchingRowId({[this._visibleColId]: value}) || raw;
|
||||
return this.tableData.findMatchingRowId({[this._visibleColId]: value}) || raw;
|
||||
}
|
||||
}
|
||||
|
||||
@ -178,7 +185,7 @@ export class ReferenceListParser extends ReferenceParser {
|
||||
// csvDecodeRow should never raise an exception
|
||||
values = csvDecodeRow(raw);
|
||||
}
|
||||
values = values.map(v => typeof v === "string" ? this._visibleColParser(v) : encodeObject(v));
|
||||
values = values.map(v => typeof v === "string" ? this.visibleColParser.cleanParse(v) : encodeObject(v));
|
||||
|
||||
if (!values.length || !raw) {
|
||||
return null; // null is the default value for a reference list column
|
||||
@ -194,7 +201,7 @@ export class ReferenceListParser extends ReferenceParser {
|
||||
}
|
||||
}
|
||||
|
||||
if (!this._tableData?.isLoaded) {
|
||||
if (!this.tableData?.isLoaded) {
|
||||
const options: { column: string, raw?: string } = {column: this._visibleColId};
|
||||
if (!(values.length === 1 && values[0] === raw)) {
|
||||
options.raw = raw;
|
||||
@ -204,7 +211,7 @@ export class ReferenceListParser extends ReferenceParser {
|
||||
|
||||
const rowIds: number[] = [];
|
||||
for (const value of values) {
|
||||
const rowId = this._tableData.findMatchingRowId({[this._visibleColId]: value});
|
||||
const rowId = this.tableData.findMatchingRowId({[this._visibleColId]: value});
|
||||
if (rowId) {
|
||||
rowIds.push(rowId);
|
||||
} else {
|
||||
@ -228,27 +235,21 @@ export const valueParserClasses: { [type: string]: typeof ValueParser } = {
|
||||
RefList: ReferenceListParser,
|
||||
};
|
||||
|
||||
const identity = (value: string) => value;
|
||||
|
||||
/**
|
||||
* Returns a function which can parse strings into values appropriate for
|
||||
* Returns a ValueParser which can parse strings into values appropriate for
|
||||
* a specific widget field or table column.
|
||||
* widgetOpts is usually the field/column's widgetOptions JSON
|
||||
* but referencing columns need more than that, see ReferenceParsingOptions above.
|
||||
*/
|
||||
export function createParserRaw(
|
||||
type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings
|
||||
): (value: string) => any {
|
||||
const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)];
|
||||
if (cls) {
|
||||
const parser = new cls(type, widgetOpts, docSettings);
|
||||
return parser.cleanParse.bind(parser);
|
||||
}
|
||||
return identity;
|
||||
): ValueParser {
|
||||
const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)] || IdentityParser;
|
||||
return new cls(type, widgetOpts, docSettings);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a function which can parse strings into values appropriate for
|
||||
* Returns a ValueParser which can parse strings into values appropriate for
|
||||
* a specific widget field or table column.
|
||||
*
|
||||
* Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
|
||||
@ -258,23 +259,46 @@ export function createParser(
|
||||
docData: DocData,
|
||||
colRef: number,
|
||||
fieldRef?: number,
|
||||
): (value: string) => any {
|
||||
): ValueParser {
|
||||
return createParserRaw(...createParserOrFormatterArguments(docData, colRef, fieldRef));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns arguments suitable for createParserRaw or createFormatter. Only for internal use.
|
||||
*
|
||||
* Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field
|
||||
* instead of the table column.
|
||||
*/
|
||||
export function createParserOrFormatterArguments(
|
||||
docData: DocData,
|
||||
colRef: number,
|
||||
fieldRef?: number,
|
||||
): [string, object, DocumentSettings] {
|
||||
const columnsTable = docData.getMetaTable('_grist_Tables_column');
|
||||
const fieldsTable = docData.getMetaTable('_grist_Views_section_field');
|
||||
const docInfoTable = docData.getMetaTable('_grist_DocInfo');
|
||||
|
||||
const col = columnsTable.getRecord(colRef)!;
|
||||
|
||||
let fieldOrCol: MetaRowRecord<'_grist_Tables_column' | '_grist_Views_section_field'> = col;
|
||||
if (fieldRef) {
|
||||
fieldOrCol = fieldsTable.getRecord(fieldRef) || col;
|
||||
}
|
||||
|
||||
const widgetOpts = safeJsonParse(fieldOrCol.widgetOptions, {});
|
||||
return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol);
|
||||
}
|
||||
|
||||
export function createParserOrFormatterArgumentsRaw(
|
||||
docData: DocData,
|
||||
type: string,
|
||||
widgetOptions: string,
|
||||
visibleColRef: number,
|
||||
): [string, object, DocumentSettings] {
|
||||
const columnsTable = docData.getMetaTable('_grist_Tables_column');
|
||||
const docInfoTable = docData.getMetaTable('_grist_DocInfo');
|
||||
|
||||
const widgetOpts = safeJsonParse(widgetOptions, {});
|
||||
|
||||
const type = col.type;
|
||||
if (isFullReferencingType(type)) {
|
||||
const vcol = columnsTable.getRecord(fieldOrCol.visibleCol);
|
||||
const vcol = columnsTable.getRecord(visibleColRef);
|
||||
widgetOpts.visibleColId = vcol?.colId || 'id';
|
||||
widgetOpts.visibleColType = vcol?.type;
|
||||
widgetOpts.visibleColWidgetOpts = safeJsonParse(vcol?.widgetOptions || '', {});
|
||||
@ -284,7 +308,7 @@ export function createParser(
|
||||
const docInfo = docInfoTable.getRecord(1);
|
||||
const docSettings = safeJsonParse(docInfo!.documentSettings, {}) as DocumentSettings;
|
||||
|
||||
return createParserRaw(type, widgetOpts, docSettings);
|
||||
return [type, widgetOpts, docSettings];
|
||||
}
|
||||
|
||||
/**
|
||||
@ -311,12 +335,12 @@ function parseColValues<T extends ColValues | BulkColValues>(
|
||||
const parser = createParser(docData, colRef);
|
||||
|
||||
// Optimisation: If there's no special parser for this column type, do nothing
|
||||
if (parser === identity) {
|
||||
if (parser instanceof IdentityParser) {
|
||||
return values;
|
||||
}
|
||||
|
||||
function parseIfString(val: any) {
|
||||
return typeof val === "string" ? parser(val) : val;
|
||||
return typeof val === "string" ? parser.cleanParse(val) : val;
|
||||
}
|
||||
|
||||
if (bulk) {
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
import {RowId} from 'app/client/models/rowset';
|
||||
import {TableData} from 'app/client/models/TableData';
|
||||
import {UIRowId} from 'app/common/UIRowId';
|
||||
import {TableData} from "./TableData";
|
||||
|
||||
/**
|
||||
* Return whether a table identified by the rowId of its metadata record, should normally be
|
||||
* hidden from the user (e.g. as an option in the page-widget picker).
|
||||
*/
|
||||
export function isHiddenTable(tablesData: TableData, tableRef: RowId): boolean {
|
||||
export function isHiddenTable(tablesData: TableData, tableRef: UIRowId): boolean {
|
||||
const tableId = tablesData.getValue(tableRef, 'tableId') as string|undefined;
|
||||
return tablesData.getValue(tableRef, 'summarySourceTable') !== 0 ||
|
||||
Boolean(tableId?.startsWith('GristHidden'));
|
||||
@ -41,6 +41,7 @@ import {schema, SCHEMA_VERSION} from 'app/common/schema';
|
||||
import {MetaRowRecord} from 'app/common/TableData';
|
||||
import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
|
||||
import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
|
||||
import {convertFromColumn} from 'app/common/ValueConverter';
|
||||
import {parseUserAction} from 'app/common/ValueParser';
|
||||
import {ParseOptions} from 'app/plugin/FileParserAPI';
|
||||
import {GristDocAPI} from 'app/plugin/GristAPI';
|
||||
@ -1717,6 +1718,12 @@ export class ActiveDoc extends EventEmitter {
|
||||
logTimes: true,
|
||||
logMeta: {docId: this._docName},
|
||||
preferredPythonVersion,
|
||||
sandboxOptions: {
|
||||
exports: {
|
||||
convertFromColumn: (...args: Parameters<ReturnType<typeof convertFromColumn>>) =>
|
||||
convertFromColumn(this.docData!)(...args)
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -87,6 +87,7 @@ const SPECIAL_ACTIONS = new Set(['InitNewDoc',
|
||||
'TransformAndFinishImport',
|
||||
'AddView',
|
||||
'CopyFromColumn',
|
||||
'ConvertFromColumn',
|
||||
'AddHiddenColumn',
|
||||
]);
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import * as log from 'app/server/lib/log';
|
||||
import {ISandboxOptions} from 'app/server/lib/NSandbox';
|
||||
|
||||
/**
|
||||
* Starting to whittle down the options used when creating a sandbox, to leave more
|
||||
@ -17,6 +18,8 @@ export interface ISandboxCreationOptions {
|
||||
importMount?: string; // if defined, make this path available read-only as "/importdir"
|
||||
|
||||
preferredPythonVersion?: '2' | '3';
|
||||
|
||||
sandboxOptions?: Partial<ISandboxOptions>;
|
||||
}
|
||||
|
||||
export interface ISandbox {
|
||||
|
||||
@ -33,7 +33,7 @@ type SandboxMethod = (...args: any[]) => any;
|
||||
* started by setting `useGristEntrypoint` (the only exception is
|
||||
* in tests) which runs grist/main.py.
|
||||
*/
|
||||
interface ISandboxOptions {
|
||||
export interface ISandboxOptions {
|
||||
command?: string; // External program or container to call to run the sandbox.
|
||||
args: string[]; // The arguments to pass to the python process.
|
||||
|
||||
@ -404,6 +404,7 @@ export class NSandboxCreator implements ISandboxCreator {
|
||||
preferredPythonVersion: this._preferredPythonVersion || options.preferredPythonVersion,
|
||||
useGristEntrypoint: true,
|
||||
importDir: options.importMount,
|
||||
...options.sandboxOptions,
|
||||
};
|
||||
return new NSandbox(translatedOptions, spawners[this._flavor]);
|
||||
}
|
||||
|
||||
@ -473,8 +473,23 @@ class BaseReferenceColumn(BaseColumn):
|
||||
.get_column_rec(self.table_id, self.col_id).visibleCol.colId
|
||||
or "id"
|
||||
)
|
||||
target_value = self._target_table.get_column(col_id)._convert_raw_value(value)
|
||||
return self._target_table.lookup_one_record(**{col_id: target_value})
|
||||
column = self._target_table.get_column(col_id)
|
||||
# `value` is an object encoded for transmission from JS to Python,
|
||||
# which is decoded to `decoded_value`.
|
||||
# `raw_value` is the kind of value that would be stored in `column`.
|
||||
# `rich_value` is the type of value used in formulas, especially with `lookupRecords`.
|
||||
# For example, for a Date column, `raw_value` is a numerical timestamp
|
||||
# and `rich_value` is a `datetime.date` object,
|
||||
# assuming `value` isn't of an invalid type.
|
||||
# However `value` could either be just a number
|
||||
# (in which case `decoded_value` would be a number as well)
|
||||
# or an encoded date (or even datetime) object like ['d', number]
|
||||
# (in which case `decoded_value` would be a `datetime.date` object,
|
||||
# which would get converted back to a number and then back to a date object again!)
|
||||
decoded_value = objtypes.decode_object(value)
|
||||
raw_value = column.convert(decoded_value)
|
||||
rich_value = column._convert_raw_value(raw_value)
|
||||
return self._target_table.lookup_one_record(**{col_id: rich_value})
|
||||
|
||||
|
||||
class ReferenceColumn(BaseReferenceColumn):
|
||||
|
||||
@ -500,6 +500,14 @@ def N(value):
|
||||
return 0
|
||||
|
||||
|
||||
def CURRENT_CONVERSION(rec):
|
||||
"""
|
||||
Special function used only when changing the type of a column.
|
||||
Doesn't work in normal formulas.
|
||||
"""
|
||||
return rec.gristHelper_Converted
|
||||
|
||||
|
||||
def NA():
|
||||
"""
|
||||
Returns the "value not available" error, `#N/A`.
|
||||
|
||||
@ -1,12 +1,16 @@
|
||||
from datetime import datetime, timedelta
|
||||
import re
|
||||
from .date import DATEADD, NOW, DTIME
|
||||
from moment_parse import MONTH_NAMES, DAY_NAMES
|
||||
|
||||
# Limit exports to schedule, so that upper-case constants like MONTH_NAMES, DAY_NAMES don't end up
|
||||
# exposed as if Excel-style functions (or break docs generation).
|
||||
__all__ = ['SCHEDULE']
|
||||
|
||||
MONTH_NAMES = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august',
|
||||
'september', 'october', 'november', 'december']
|
||||
# Regex list of lowercase weekdays with characters after the first three made optional
|
||||
DAY_NAMES = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday']
|
||||
|
||||
def SCHEDULE(schedule, start=None, count=10, end=None):
|
||||
"""
|
||||
Returns the list of `datetime` objects generated according to the `schedule` string. Starts at
|
||||
|
||||
@ -4,7 +4,6 @@ import marshal
|
||||
from time import time
|
||||
import bisect
|
||||
import os
|
||||
import moment_parse
|
||||
import iso8601
|
||||
import six
|
||||
from six.moves import zip
|
||||
@ -13,7 +12,6 @@ from six.moves import zip
|
||||
ZoneRecord = namedtuple("ZoneRecord", ("name", "abbrs", "offsets", "untils"))
|
||||
|
||||
# moment.py mirrors core functionality of moment-timezone.js
|
||||
# moment.py includes function parse, located and documented in moment_parse.py
|
||||
# Documentation: http://momentjs.com/timezone/docs/
|
||||
|
||||
EPOCH = datetime(1970, 1, 1)
|
||||
@ -67,10 +65,6 @@ def date_to_ts(date, timezone=None):
|
||||
ts = (date - DATE_EPOCH).total_seconds()
|
||||
return ts if not timezone else ts - timezone.offset(ts * 1000).total_seconds()
|
||||
|
||||
# Calls parse from moment_parse.py
|
||||
def parse(date_string, parse_format, zonelabel='UTC'):
|
||||
return moment_parse.parse(date_string, parse_format, zonelabel)
|
||||
|
||||
# Parses a datetime in the ISO format, YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM. Most parts are optional;
|
||||
# see https://pypi.org/project/iso8601/ for details. Returns a timestamp in seconds.
|
||||
def parse_iso(date_string, timezone=None):
|
||||
|
||||
@ -1,161 +0,0 @@
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
import moment
|
||||
|
||||
# Regex list of lowercase months with characters after the first three made optional
|
||||
MONTH_NAMES = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august',
|
||||
'september', 'october', 'november', 'december']
|
||||
MONTHS = [m[:3]+"(?:"+m[3:]+")?" if len(m) > 3 else m[:3] for m in MONTH_NAMES]
|
||||
# Regex list of lowercase weekdays with characters after the first three made optional
|
||||
DAY_NAMES = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday']
|
||||
WEEKDAYS = [d[:3]+"(?:"+d[3:]+")?" for d in DAY_NAMES]
|
||||
|
||||
# Acceptable format tokens mapped to what they should match in the date string
|
||||
# Ordered so that larger configurations are matched first
|
||||
DATE_TOKENS = OrderedDict([
|
||||
("HH", r"(?P<H>\d{1,2})"), # 24 hr
|
||||
("H", r"(?P<H>\d{1,2})"),
|
||||
("hh", r"(?P<h>\d{1,2})"), # 12 hr
|
||||
("h", r"(?P<h>\d{1,2})"),
|
||||
("mm", r"(?P<m>\d{1,2})"), # min
|
||||
("m", r"(?P<m>\d{1,2})"),
|
||||
("A", r"(?P<A>[ap]m?)"), # am/pm
|
||||
("a", r"(?P<A>[ap]m?)"),
|
||||
("ss", r"(?P<s>\d{1,2})"), # sec
|
||||
("s", r"(?P<s>\d{1,2})"),
|
||||
("SSSSSS", r"(?P<S>\d{1,6})"), # fractional second
|
||||
("SSSSS", r"(?P<S>\d{1,6})"),
|
||||
("SSSS", r"(?P<S>\d{1,6})"),
|
||||
("SSS", r"(?P<S>\d{1,6})"),
|
||||
("SS", r"(?P<S>\d{1,6})"),
|
||||
("S", r"(?P<S>\d{1,6})"),
|
||||
("YYYY", r"(?P<YY>\d{4}|\d{2})"), # 4 or 2 digit year
|
||||
("YY", r"(?P<YY>\d{2})"), # 2 digit year
|
||||
("MMMM", r"(?P<MMM>" + ("|".join(MONTHS)) + ")"), # month name, abbr or not
|
||||
("MMM", r"(?P<MMM>" + ("|".join(MONTHS)) + ")"),
|
||||
("MM", r"(?P<M>\d{1,2})"), # month num
|
||||
("M", r"(?P<M>\d{1,2})"),
|
||||
("DD", r"(?P<D>\d{1,2})"), # day num
|
||||
("Do", r"(?P<D>\d{1,2})(st|nd|rd|th)"),
|
||||
("D", r"(?P<D>\d{1,2})"),
|
||||
("dddd", r"(" + ("|".join(WEEKDAYS)) + ")"), # day name, abbr or not (ignored)
|
||||
("ddd", r"(" + ("|".join(WEEKDAYS)) + ")")
|
||||
])
|
||||
DATE_TOKENS_REGEX = re.compile("("+("|".join(DATE_TOKENS))+")")
|
||||
|
||||
# List of separators to replace and match any standard date/time separators
|
||||
SEP = r"[\s/.\-:,]*"
|
||||
SEP_REGEX = re.compile(SEP)
|
||||
SEP_REPLACEMENT = SEP.replace("\\", "\\\\")
|
||||
|
||||
# Maps date parse format to compile regex
|
||||
FORMAT_CACHE = {}
|
||||
|
||||
# Parses date_string using parse_format in the style of moment.js
|
||||
# See: http://momentjs.com/docs/#/parsing
|
||||
# Supports the following tokens:
|
||||
# H HH 0..23 24 hour time
|
||||
# h hh 1..12 12 hour time used with a A.
|
||||
# a A am pm Post or ante meridiem
|
||||
# m mm 0..59 Minutes
|
||||
# s ss 0..59 Seconds
|
||||
# S SS SSS 0..999 Fractional seconds
|
||||
# YYYY 2014 4 or 2 digit year
|
||||
# YY 14 2 digit year
|
||||
# M MM 1..12 Month number
|
||||
# MMM MMMM Jan..December Month name in locale set by moment.locale()
|
||||
# D DD 1..31 Day of month
|
||||
# Do 1st..31st Day of month with ordinal
|
||||
def parse(date_string, parse_format, zonelabel='UTC', override_current_date=None):
|
||||
"""Parse a date string via a moment.js style parse format and a timezone string.
|
||||
Supported tokens are documented above. Returns seconds since epoch"""
|
||||
|
||||
if parse_format in FORMAT_CACHE:
|
||||
# Check if parse_format has been cache, and retrieve if so
|
||||
parser = FORMAT_CACHE[parse_format]
|
||||
else:
|
||||
# e.g. "MM-YY" -> "(?P<mm>\d{1,2})-(?P<yy>\d{2})"
|
||||
# Note that DATE_TOKENS is ordered so that the longer letter chains are recognized first
|
||||
tokens = DATE_TOKENS_REGEX.split(parse_format)
|
||||
tokens = [DATE_TOKENS[t] if t in DATE_TOKENS else SEP_REGEX.sub(SEP_REPLACEMENT, t)
|
||||
for t in tokens]
|
||||
|
||||
# Compile new token string ignoring case (for month names)
|
||||
parser = re.compile(''.join(tokens), re.I)
|
||||
FORMAT_CACHE[parse_format] = parser
|
||||
|
||||
match = parser.match(date_string)
|
||||
|
||||
# Throw error if matching failed
|
||||
if match is None:
|
||||
raise Exception("Failed to parse %s with %s" % (date_string, parse_format))
|
||||
|
||||
# Create datetime from the results of parsing
|
||||
current_date = override_current_date or moment.CURRENT_DATE
|
||||
m = match.groupdict()
|
||||
dt = datetime(
|
||||
year=getYear(m, current_date.year),
|
||||
month=getMonth(m, current_date.month),
|
||||
day=int(m['D']) if ('D' in m) else current_date.day,
|
||||
hour=getHour(m),
|
||||
minute=int(m['m']) if ('m' in m) else 0,
|
||||
second=int(m['s']) if ('s' in m) else 0,
|
||||
microsecond=getMicrosecond(m)
|
||||
)
|
||||
|
||||
# Parses the datetime with the given timezone to return the seconds since EPOCH
|
||||
return moment.tz(dt, zonelabel).timestamp_s()
|
||||
|
||||
|
||||
def getYear(match_dict, current_year):
|
||||
if 'YYYY' in match_dict:
|
||||
return int(match_dict['YYYY'])
|
||||
elif 'YY' in match_dict:
|
||||
match = match_dict['YY']
|
||||
if len(match) == 2:
|
||||
# Must guess on the century, choose so the result is closest to the current year
|
||||
# The first year that could be meant by YY is the current year - 50.
|
||||
first = current_year - 50
|
||||
# We are seeking k such that 100k + YY is between first and first + 100.
|
||||
# first <= 100k + YY < first + 100
|
||||
# 0 <= 100k + YY - first < 100
|
||||
# The value inside the comparison operators is precisely (YY - first) % 100.
|
||||
# So we can calculate the century 100k as (YY - first) % 100 - (YY - first).
|
||||
return first + (int(match) - first) % 100
|
||||
else:
|
||||
return int(match)
|
||||
else:
|
||||
return current_year
|
||||
|
||||
def getMonth(match_dict, current_month):
|
||||
if 'M' in match_dict:
|
||||
return int(match_dict['M'])
|
||||
elif 'MMM' in match_dict:
|
||||
return lazy_index(MONTHS, match_dict['MMM'][:3].lower()) + 1
|
||||
else:
|
||||
return current_month
|
||||
|
||||
def getHour(match_dict):
|
||||
if 'H' in match_dict:
|
||||
return int(match_dict['H'])
|
||||
elif 'h' in match_dict:
|
||||
hr = int(match_dict['h']) % 12
|
||||
merid = 12 if 'A' in match_dict and match_dict['A'][0] == "p" else 0
|
||||
return hr + merid
|
||||
else:
|
||||
return 0
|
||||
|
||||
def getMicrosecond(match_dict):
|
||||
if 'S' in match_dict:
|
||||
match = match_dict['S']
|
||||
return int(match + ("0"*(6-len(match))) if len(match) < 6 else match[:6])
|
||||
else:
|
||||
return 0
|
||||
|
||||
# Gets the index of the first string from iter that starts with startswith
|
||||
def lazy_index(l, startswith, missing=None):
|
||||
for i, token in enumerate(l):
|
||||
if token[:len(startswith)] == startswith:
|
||||
return i
|
||||
return missing
|
||||
@ -1,7 +1,6 @@
|
||||
from datetime import datetime, date, timedelta
|
||||
import unittest
|
||||
import moment
|
||||
import moment_parse
|
||||
|
||||
# Helpful strftime() format that imcludes all parts of the date including the time zone.
|
||||
fmt = "%Y-%m-%d %H:%M:%S %Z"
|
||||
@ -60,78 +59,6 @@ class TestMoment(unittest.TestCase):
|
||||
[datetime(2037, 11, 1, 1, 0, 0, 0), 2140675200000, "PDT", 420, 1, 0],
|
||||
]
|
||||
|
||||
parse_samples = [
|
||||
# Basic set
|
||||
['MM-DD-YYYY', '12-02-1999', 944092800.000000],
|
||||
['DD-MM-YYYY', '12-02-1999', 918777600.000000],
|
||||
['DD/MM/YYYY', '12/02/1999', 918777600.000000],
|
||||
['DD_MM_YYYY', '12_02_1999', 918777600.000000],
|
||||
['DD:MM:YYYY', '12:02:1999', 918777600.000000],
|
||||
['D-M-YY', '2-2-99', 917913600.000000],
|
||||
['YY', '99', 922060800.000000],
|
||||
['DD-MM-YYYY h:m:s', '12-02-1999 2:45:10', 918787510.000000],
|
||||
['DD-MM-YYYY h:m:s a', '12-02-1999 2:45:10 am', 918787510.000000],
|
||||
['DD-MM-YYYY h:m:s a', '12-02-1999 2:45:10 pm', 918830710.000000],
|
||||
['h:mm a', '12:00 pm', 1458648000.000000],
|
||||
['h:mm a', '12:30 pm', 1458649800.000000],
|
||||
['h:mm a', '12:00 am', 1458604800.000000],
|
||||
['h:mm a', '12:30 am', 1458606600.000000],
|
||||
['HH:mm', '12:00', 1458648000.000000],
|
||||
['YYYY-MM-DDTHH:mm:ss', '2011-11-11T11:11:11', 1321009871.000000],
|
||||
['ddd MMM DD HH:mm:ss YYYY', 'Tue Apr 07 22:52:51 2009', 1239144771.000000],
|
||||
['ddd MMMM DD HH:mm:ss YYYY', 'Tue April 07 22:52:51 2009', 1239144771.000000],
|
||||
['HH:mm:ss', '12:00:00', 1458648000.000000],
|
||||
['HH:mm:ss', '12:30:00', 1458649800.000000],
|
||||
['HH:mm:ss', '00:00:00', 1458604800.000000],
|
||||
['HH:mm:ss S', '00:30:00 1', 1458606600.100000],
|
||||
['HH:mm:ss SS', '00:30:00 12', 1458606600.120000],
|
||||
['HH:mm:ss SSS', '00:30:00 123', 1458606600.123000],
|
||||
['HH:mm:ss S', '00:30:00 7', 1458606600.700000],
|
||||
['HH:mm:ss SS', '00:30:00 78', 1458606600.780000],
|
||||
['HH:mm:ss SSS', '00:30:00 789', 1458606600.789000],
|
||||
|
||||
# Dropped m
|
||||
['MM/DD/YYYY h:m:s a', '05/1/2012 12:25:00 p', 1335875100.000000],
|
||||
['MM/DD/YYYY h:m:s a', '05/1/2012 12:25:00 a', 1335831900.000000],
|
||||
|
||||
# 2 digit year with YYYY
|
||||
['D/M/YYYY', '9/2/99', 918518400.000000],
|
||||
['D/M/YYYY', '9/2/1999', 918518400.000000],
|
||||
['D/M/YYYY', '9/2/66', -122860800.000000],
|
||||
['D/M/YYYY', '9/2/65', 3001363200.000000],
|
||||
|
||||
# No separators
|
||||
['MMDDYYYY', '12021999', 944092800.000000],
|
||||
['DDMMYYYY', '12021999', 918777600.000000],
|
||||
['YYYYMMDD', '19991202', 944092800.000000],
|
||||
['DDMMMYYYY', '10Sep2001', 1000080000.000000],
|
||||
|
||||
# Error forgiveness
|
||||
['MM/DD/YYYY', '12-02-1999', 944092800.000000],
|
||||
['DD/MM/YYYY', '12/02 /1999', 918777600.000000],
|
||||
['DD:MM:YYYY', '12:02 :1999', 918777600.000000],
|
||||
['D-M-YY', '2 2 99', 917913600.000000],
|
||||
['DD-MM-YYYY h:m:s', '12-02-1999 2:45:10.00', 918787510.000000],
|
||||
['h:mm a', '12:00pm', 1458648000.000000],
|
||||
['HH:mm', '1200', 1458648000.000000],
|
||||
['dddd MMMM DD HH:mm:ss YYYY', 'Tue Apr 7 22:52:51 2009', 1239144771.000000],
|
||||
['ddd MMM DD HH:mm:ss YYYY', 'Tuesday April 7 22:52:51 2009', 1239144771.000000],
|
||||
['ddd MMM Do HH:mm:ss YYYY', 'Tuesday April 7th 22:52:51 2009', 1239144771.000000]
|
||||
]
|
||||
|
||||
parse_timezone_samples = [
|
||||
# Timezone corner cases
|
||||
['MM-DD-YYYY h:ma', '3-13-2016 1:59am', 'America/New_York', 1457852340], # EST
|
||||
['MM-DD-YYYY h:ma', '3-13-2016 2:00am', 'America/New_York', 1457848800], # Invalid, -1hr
|
||||
['MM-DD-YYYY h:ma', '3-13-2016 2:59am', 'America/New_York', 1457852340], # Invalid, -1hr
|
||||
['MM-DD-YYYY h:ma', '3-13-2016 3:00am', 'America/New_York', 1457852400], # EDT
|
||||
['MM-DD-YYYY h:ma', '3-13-2016 1:59am', 'America/Los_Angeles', 1457863140], # PST
|
||||
['MM-DD-YYYY h:ma', '3-13-2016 2:00am', 'America/Los_Angeles', 1457859600], # Invalid, -1hr
|
||||
['MM-DD-YYYY h:ma', '3-13-2016 2:59am', 'America/Los_Angeles', 1457863140], # Invalid, -1hr
|
||||
['MM-DD-YYYY h:ma', '3-13-2016 3:00am', 'America/Los_Angeles', 1457863200] # PDT
|
||||
]
|
||||
|
||||
|
||||
def assertMatches(self, data_entry, moment_obj):
|
||||
date, timestamp, abbr, offset, hour, minute = data_entry
|
||||
dt = moment_obj.datetime()
|
||||
@ -183,12 +110,6 @@ class TestMoment(unittest.TestCase):
|
||||
self.assertEqual(dt.tzname(), abbr)
|
||||
self.assertEqual(dt.utcoffset(), timedelta(minutes=-offset))
|
||||
|
||||
def test_parse(self):
|
||||
for s in self.parse_samples:
|
||||
self.assertEqual(moment_parse.parse(s[1], s[0], 'UTC', date(2016, 3, 22)), s[2])
|
||||
for s in self.parse_timezone_samples:
|
||||
self.assertEqual(moment_parse.parse(s[1], s[0], s[2], date(2016, 3, 22)), s[3])
|
||||
|
||||
def test_ts_to_dt(self):
|
||||
# Verify that ts_to_dt works as expected.
|
||||
value_sec = 1426291200 # 2015-03-14 00:00:00 in UTC
|
||||
|
||||
@ -64,7 +64,7 @@ class TestRefListRelation(test_engine.EngineTestCase):
|
||||
self.apply_user_action(
|
||||
['AddColumn', 'TableC', 'gristHelper_Transform', {
|
||||
"type": 'Ref:TableA', "isFormula": True,
|
||||
"formula": "grist.Reference.typeConvert($ColB, TableA, 'ColA')", "visibleCol": 2,
|
||||
"formula": "TableA.lookupOne(ColA=$ColB)", "visibleCol": 2,
|
||||
}])
|
||||
self.apply_user_action(
|
||||
['SetDisplayFormula', 'TableC', None, 7, '$gristHelper_Transform.ColA'])
|
||||
|
||||
@ -1037,7 +1037,13 @@ class UserActions(object):
|
||||
if not clean_colinfo["isFormula"]:
|
||||
raise ValueError("AddColumn: cannot add a non-formula column to a summary table")
|
||||
|
||||
transform = col_id is not None and col_id.startswith('gristHelper_Transform')
|
||||
transform = (
|
||||
col_id is not None and
|
||||
col_id.startswith((
|
||||
'gristHelper_Transform',
|
||||
'gristHelper_Converted',
|
||||
))
|
||||
)
|
||||
|
||||
if transform:
|
||||
# Delete any currently existing transform columns with the same id
|
||||
@ -1256,6 +1262,29 @@ class UserActions(object):
|
||||
finally:
|
||||
self._engine.out_actions.undo.append(mod_action)
|
||||
|
||||
@useraction
|
||||
def ConvertFromColumn(self, table_id, src_col_id, dst_col_id, typ, widgetOptions, visibleColRef):
|
||||
from sandbox import call_external
|
||||
table = self._engine.tables[table_id]
|
||||
src_col = self._docmodel.get_column_rec(table_id, src_col_id)
|
||||
src_column = table.get_column(src_col_id)
|
||||
row_ids = list(table.row_ids)
|
||||
src_values = [encode_object(src_column.raw_get(r)) for r in row_ids]
|
||||
display_values = None
|
||||
if src_col.displayCol:
|
||||
display_col = table.get_column(src_col.displayCol.colId)
|
||||
display_values = [encode_object(display_col.raw_get(r)) for r in row_ids]
|
||||
converted_values = call_external(
|
||||
"convertFromColumn",
|
||||
src_col.id,
|
||||
typ,
|
||||
widgetOptions,
|
||||
visibleColRef,
|
||||
src_values,
|
||||
display_values,
|
||||
)
|
||||
self.ModifyColumn(table_id, dst_col_id, {"type": typ})
|
||||
self.BulkUpdateRecord(table_id, row_ids, {dst_col_id: converted_values})
|
||||
|
||||
@useraction
|
||||
def CopyFromColumn(self, table_id, src_col_id, dst_col_id, widgetOptions):
|
||||
|
||||
@ -138,17 +138,6 @@ class BaseColumnType(object):
|
||||
return objtypes.safe_repr(value_to_convert)
|
||||
|
||||
|
||||
# This is a user-facing method, hence the camel-case naming, as for `lookupRecords` and such.
|
||||
@classmethod
|
||||
def typeConvert(cls, value):
|
||||
"""
|
||||
Convert a value from a different type to something that this type can accept, as when
|
||||
explicitly converting a column type. Note that usual conversion (such as converting numbers to
|
||||
strings or vice versa) will still apply to the returned value.
|
||||
"""
|
||||
return value
|
||||
|
||||
|
||||
class Text(BaseColumnType):
|
||||
"""
|
||||
Text is the type for a field holding string (text) data.
|
||||
@ -180,18 +169,6 @@ class Text(BaseColumnType):
|
||||
def is_right_type(cls, value):
|
||||
return isinstance(value, (six.string_types, NoneType))
|
||||
|
||||
@classmethod
|
||||
def typeConvert(cls, value):
|
||||
if value is None:
|
||||
# When converting NULLs (that typically show up as a plain empty cell for Numeric or Date
|
||||
# columns) to Text, it makes more sense to end up with a plain blank text cell.
|
||||
return ''
|
||||
elif isinstance(value, bool):
|
||||
# Normalize True/False to true/false (Toggle columns use true/false).
|
||||
return str(value).lower()
|
||||
else:
|
||||
return value
|
||||
|
||||
|
||||
class Blob(BaseColumnType):
|
||||
"""
|
||||
@ -302,13 +279,6 @@ class Date(Numeric):
|
||||
def is_right_type(cls, value):
|
||||
return isinstance(value, (float, six.integer_types, NoneType))
|
||||
|
||||
@classmethod
|
||||
def typeConvert(cls, value, date_format, timezone='UTC'): # pylint: disable=arguments-differ
|
||||
# Note: the timezone argument is used in DateTime conversions, allows sharing this method.
|
||||
try:
|
||||
return moment.parse(value, date_format, timezone)
|
||||
except Exception:
|
||||
return value
|
||||
|
||||
class DateTime(Date):
|
||||
"""
|
||||
@ -370,21 +340,6 @@ class ChoiceList(BaseColumnType):
|
||||
return value is None or (isinstance(value, (tuple, list)) and
|
||||
all(isinstance(item, six.string_types) for item in value))
|
||||
|
||||
@classmethod
|
||||
def typeConvert(cls, value):
|
||||
if value is None:
|
||||
return value
|
||||
if isinstance(value, six.string_types) and not value.startswith('['):
|
||||
# Try to parse as CSV. If this doesn't work, we'll still try usual conversions later.
|
||||
try:
|
||||
tags = next(csv.reader([value]))
|
||||
return tuple(t.strip() for t in tags if t.strip())
|
||||
except Exception:
|
||||
pass
|
||||
if not isinstance(value, (tuple, list)):
|
||||
value = [Choice.typeConvert(value)]
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def toString(cls, value):
|
||||
if isinstance(value, (tuple, list)):
|
||||
@ -458,13 +413,6 @@ class Reference(Id):
|
||||
def typename(cls):
|
||||
return "Ref"
|
||||
|
||||
@classmethod
|
||||
def typeConvert(cls, value, ref_table, visible_col=None): # pylint: disable=arguments-differ
|
||||
if value and ref_table and visible_col:
|
||||
return ref_table.lookupOne(**{visible_col: value}) or six.text_type(value)
|
||||
else:
|
||||
return value
|
||||
|
||||
|
||||
class ReferenceList(BaseColumnType):
|
||||
"""
|
||||
@ -500,15 +448,6 @@ class ReferenceList(BaseColumnType):
|
||||
return value is None or (isinstance(value, list) and
|
||||
all(Reference.is_right_type(val) for val in value))
|
||||
|
||||
@classmethod
|
||||
def typeConvert(cls, value, ref_table, visible_col=None): # noqa # pylint: disable=arguments-differ
|
||||
# TODO this is based on Reference.typeConvert.
|
||||
# It doesn't make much sense as a conversion but I don't know what would
|
||||
if value and ref_table and visible_col:
|
||||
return ref_table.lookupRecords(**{visible_col: value}) or six.text_type(value)
|
||||
else:
|
||||
return value
|
||||
|
||||
|
||||
class Attachments(ReferenceList):
|
||||
"""
|
||||
@ -516,8 +455,3 @@ class Attachments(ReferenceList):
|
||||
"""
|
||||
def __init__(self):
|
||||
super(Attachments, self).__init__('_grist_Attachments')
|
||||
|
||||
@classmethod
|
||||
def typeConvert(cls, value): # noqa # pylint: disable=arguments-differ
|
||||
# Don't use ReferenceList.typeConvert which is called with a different number of arguments
|
||||
return value
|
||||
|
||||
@ -1053,6 +1053,10 @@ export async function setType(type: RegExp, options: {skipWait?: boolean} = {})
|
||||
if (!options.skipWait) { await waitForServer(); }
|
||||
}
|
||||
|
||||
export async function applyTypeTransform() {
|
||||
await driver.findContent('.type_transform_prompt button', /Apply/).click();
|
||||
}
|
||||
|
||||
export async function isMac(): Promise<boolean> {
|
||||
return /Darwin|Mac|iPod|iPhone|iPad/i.test((await driver.getCapabilities()).get('platform'));
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user