mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) Parsing multiple values in reflists, parsing refs without table data in client
Summary: Added a new object type code `l` (for lookup) which can be used in user actions as a temporary cell value in ref[list] columns and is immediately converted to a row ID in the data engine. The value contains the original raw string (to be used as alt text), the column ID to lookup (typically the visible column) and one or more values to lookup. For reflists, valueParser now tries parsing the string first as JSON, then as a CSV row, and applies the visible column parsed to each item. Both ref and reflists columns no longer format the parsed value when there's no matching reference, the original unparsed string is used as alttext instead. Test Plan: Added another table "Multi-References" to CopyPaste test. Made that table and the References table test with and without table data loaded in the browser. Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D3118
This commit is contained in:
parent
b6dd066b7f
commit
ecb30eebb8
@ -2,7 +2,7 @@ import { DocData } from 'app/client/models/DocData';
|
|||||||
import {ColumnRec} from 'app/client/models/entities/ColumnRec';
|
import {ColumnRec} from 'app/client/models/entities/ColumnRec';
|
||||||
import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec';
|
import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec';
|
||||||
import {SearchFunc, TableData} from 'app/client/models/TableData';
|
import {SearchFunc, TableData} from 'app/client/models/TableData';
|
||||||
import { getReferencedTableId } from 'app/common/gristTypes';
|
import {getReferencedTableId, isRefListType} from 'app/common/gristTypes';
|
||||||
import {BaseFormatter} from 'app/common/ValueFormatter';
|
import {BaseFormatter} from 'app/common/ValueFormatter';
|
||||||
import isEqual = require('lodash/isEqual');
|
import isEqual = require('lodash/isEqual');
|
||||||
|
|
||||||
@ -15,6 +15,7 @@ export class ReferenceUtils {
|
|||||||
public readonly formatter: BaseFormatter;
|
public readonly formatter: BaseFormatter;
|
||||||
public readonly visibleColModel: ColumnRec;
|
public readonly visibleColModel: ColumnRec;
|
||||||
public readonly visibleColId: string;
|
public readonly visibleColId: string;
|
||||||
|
public readonly isRefList: boolean;
|
||||||
|
|
||||||
constructor(public readonly field: ViewFieldRec, docData: DocData) {
|
constructor(public readonly field: ViewFieldRec, docData: DocData) {
|
||||||
// Note that this constructor is called inside ViewFieldRec.valueParser, a ko.pureComputed,
|
// Note that this constructor is called inside ViewFieldRec.valueParser, a ko.pureComputed,
|
||||||
@ -36,50 +37,83 @@ export class ReferenceUtils {
|
|||||||
this.formatter = field.createVisibleColFormatter();
|
this.formatter = field.createVisibleColFormatter();
|
||||||
this.visibleColModel = field.visibleColModel();
|
this.visibleColModel = field.visibleColModel();
|
||||||
this.visibleColId = this.visibleColModel.colId() || 'id';
|
this.visibleColId = this.visibleColModel.colId() || 'id';
|
||||||
|
this.isRefList = isRefListType(colType);
|
||||||
}
|
}
|
||||||
|
|
||||||
public parseValue(value: any): number | string {
|
public parseReference(
|
||||||
if (!value) {
|
raw: string, value: unknown
|
||||||
return 0; // This is the default value for a reference column.
|
): number | string | ['l', unknown, {raw?: string, column: string}] {
|
||||||
|
if (!value || !raw) {
|
||||||
|
return 0; // default value for a reference column
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.visibleColId === 'id') {
|
if (this.visibleColId === 'id') {
|
||||||
const n = Number(value);
|
const n = Number(value);
|
||||||
if (
|
if (Number.isInteger(n)) {
|
||||||
n > 0 &&
|
value = n;
|
||||||
Number.isInteger(n) &&
|
} else {
|
||||||
!(
|
return raw;
|
||||||
this.tableData.isLoaded &&
|
|
||||||
!this.tableData.hasRowId(n)
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
return n;
|
|
||||||
}
|
}
|
||||||
return String(value);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let searchFunc: SearchFunc;
|
if (!this.tableData.isLoaded) {
|
||||||
if (typeof value === 'string') {
|
const options: {column: string, raw?: string} = {column: this.visibleColId};
|
||||||
searchFunc = (v: any) => {
|
if (value !== raw) {
|
||||||
const formatted = this.formatter.formatAny(v);
|
options.raw = raw;
|
||||||
return nocaseEqual(formatted, value);
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
searchFunc = (v: any) => isEqual(v, value);
|
|
||||||
}
|
}
|
||||||
|
return ['l', value, options];
|
||||||
|
}
|
||||||
|
|
||||||
|
const searchFunc: SearchFunc = (v: any) => isEqual(v, value);
|
||||||
const matches = this.tableData.columnSearch(this.visibleColId, searchFunc, 1);
|
const matches = this.tableData.columnSearch(this.visibleColId, searchFunc, 1);
|
||||||
if (matches.length > 0) {
|
if (matches.length > 0) {
|
||||||
return matches[0];
|
return matches[0];
|
||||||
} else {
|
} else {
|
||||||
// There's no matching value in the visible column, i.e. this is not a valid reference.
|
// There's no matching value in the visible column, i.e. this is not a valid reference.
|
||||||
// We need to return a string which will become AltText.
|
// We need to return a string which will become AltText.
|
||||||
// Can't return `value` directly because it may be a number (if visibleCol is a numeric or date column)
|
return raw;
|
||||||
// which would be interpreted as a row ID, i.e. a valid reference.
|
|
||||||
// So instead we format the parsed value in the style of visibleCol.
|
|
||||||
return this.formatter.formatAny(value);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public parseReferenceList(
|
||||||
|
raw: string, values: unknown[]
|
||||||
|
): ['L', ...number[]] | null | string | ['l', unknown[], {raw?: string, column: string}] {
|
||||||
|
if (!values.length || !raw) {
|
||||||
|
return null; // default value for a reference list column
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.visibleColId === 'id') {
|
||||||
|
const numbers = values.map(Number);
|
||||||
|
if (numbers.every(Number.isInteger)) {
|
||||||
|
values = numbers;
|
||||||
|
} else {
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.tableData.isLoaded) {
|
||||||
|
const options: {column: string, raw?: string} = {column: this.visibleColId};
|
||||||
|
if (!(values.length === 1 && values[0] === raw)) {
|
||||||
|
options.raw = raw;
|
||||||
|
}
|
||||||
|
return ['l', values, options];
|
||||||
|
}
|
||||||
|
|
||||||
|
const rowIds: number[] = [];
|
||||||
|
for (const value of values) {
|
||||||
|
const searchFunc: SearchFunc = (v: any) => isEqual(v, value);
|
||||||
|
const matches = this.tableData.columnSearch(this.visibleColId, searchFunc, 1);
|
||||||
|
if (matches.length > 0) {
|
||||||
|
rowIds.push(matches[0]);
|
||||||
|
} else {
|
||||||
|
// There's no matching value in the visible column, i.e. this is not a valid reference.
|
||||||
|
// We need to return a string which will become AltText.
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ['L', ...rowIds];
|
||||||
|
}
|
||||||
|
|
||||||
public idToText(value: unknown) {
|
public idToText(value: unknown) {
|
||||||
if (typeof value === 'number') {
|
if (typeof value === 'number') {
|
||||||
return this.formatter.formatAny(this.tableData.getValue(value, this.visibleColId));
|
return this.formatter.formatAny(this.tableData.getValue(value, this.visibleColId));
|
||||||
|
@ -2,8 +2,9 @@ import { ReferenceUtils } from 'app/client/lib/ReferenceUtils';
|
|||||||
import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel';
|
import {ColumnRec, DocModel, IRowModel, refRecord, ViewSectionRec} from 'app/client/models/DocModel';
|
||||||
import * as modelUtil from 'app/client/models/modelUtil';
|
import * as modelUtil from 'app/client/models/modelUtil';
|
||||||
import * as UserType from 'app/client/widgets/UserType';
|
import * as UserType from 'app/client/widgets/UserType';
|
||||||
|
import {csvDecodeRow} from 'app/common/csvFormat';
|
||||||
import {DocumentSettings} from 'app/common/DocumentSettings';
|
import {DocumentSettings} from 'app/common/DocumentSettings';
|
||||||
import { isFullReferencingType, isRefListType } from 'app/common/gristTypes';
|
import {isFullReferencingType} from 'app/common/gristTypes';
|
||||||
import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter';
|
import {BaseFormatter, createFormatter} from 'app/common/ValueFormatter';
|
||||||
import {createParser} from 'app/common/ValueParser';
|
import {createParser} from 'app/common/ValueParser';
|
||||||
import {Computed, fromKo} from 'grainjs';
|
import {Computed, fromKo} from 'grainjs';
|
||||||
@ -192,25 +193,25 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void
|
|||||||
const vcol = this.visibleColModel();
|
const vcol = this.visibleColModel();
|
||||||
const vcolParser = createParser(vcol.type(), vcol.widgetOptionsJson(), docSettings);
|
const vcolParser = createParser(vcol.type(), vcol.widgetOptionsJson(), docSettings);
|
||||||
const refUtils = new ReferenceUtils(this, docModel.docData); // uses several more observables immediately
|
const refUtils = new ReferenceUtils(this, docModel.docData); // uses several more observables immediately
|
||||||
return (s: string) => {
|
if (!refUtils.isRefList) {
|
||||||
const result = refUtils.parseValue(vcolParser(s));
|
return (s: string) => refUtils.parseReference(s, vcolParser(s));
|
||||||
// If `result` is a number that means it successfully parsed a reference (row ID).
|
|
||||||
// For a reflist we need to wrap that row ID in a list.
|
|
||||||
// Otherwise `result` is a string meaning it couldn't be parsed
|
|
||||||
// and it will be saved as AltText (i.e. invalid for the column type).
|
|
||||||
// We don't try to parse multiple references from a single string.
|
|
||||||
if (isRefListType(type) && typeof result === "number") {
|
|
||||||
if (result > 0) {
|
|
||||||
return ['L', result];
|
|
||||||
} else {
|
} else {
|
||||||
// parseValue returns 0 sometimes because that's the default for reference columns.
|
return (s: string) => {
|
||||||
// The default for a reflist column is null.
|
let values: any[] | null;
|
||||||
return null;
|
try {
|
||||||
|
values = JSON.parse(s);
|
||||||
|
} catch {
|
||||||
|
values = null;
|
||||||
}
|
}
|
||||||
|
if (!Array.isArray(values)) {
|
||||||
|
// csvDecodeRow should never raise an exception
|
||||||
|
values = csvDecodeRow(s);
|
||||||
}
|
}
|
||||||
return result;
|
values = values.map(v => typeof v === "string" ? vcolParser(v) : v);
|
||||||
|
return refUtils.parseReferenceList(s, values);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// The widgetOptions to read and write: either the column's or the field's own.
|
// The widgetOptions to read and write: either the column's or the field's own.
|
||||||
|
@ -6,6 +6,7 @@ import * as t from "ts-interface-checker";
|
|||||||
|
|
||||||
export const GristObjCode = t.enumtype({
|
export const GristObjCode = t.enumtype({
|
||||||
"List": "L",
|
"List": "L",
|
||||||
|
"LookUp": "l",
|
||||||
"Dict": "O",
|
"Dict": "O",
|
||||||
"DateTime": "D",
|
"DateTime": "D",
|
||||||
"Date": "d",
|
"Date": "d",
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
// Letter codes for CellValue types encoded as [code, args...] tuples.
|
// Letter codes for CellValue types encoded as [code, args...] tuples.
|
||||||
export const enum GristObjCode {
|
export const enum GristObjCode {
|
||||||
List = 'L',
|
List = 'L',
|
||||||
|
LookUp = 'l',
|
||||||
Dict = 'O',
|
Dict = 'O',
|
||||||
DateTime = 'D',
|
DateTime = 'D',
|
||||||
Date = 'd',
|
Date = 'd',
|
||||||
|
@ -158,8 +158,15 @@ class BaseColumn(object):
|
|||||||
raise raw.error
|
raise raw.error
|
||||||
else:
|
else:
|
||||||
raise objtypes.CellError(self.table_id, self.col_id, row_id, raw.error)
|
raise objtypes.CellError(self.table_id, self.col_id, row_id, raw.error)
|
||||||
|
|
||||||
|
return self._convert_raw_value(raw)
|
||||||
|
|
||||||
|
def _convert_raw_value(self, raw):
|
||||||
if self.type_obj.is_right_type(raw):
|
if self.type_obj.is_right_type(raw):
|
||||||
return self._make_rich_value(raw)
|
return self._make_rich_value(raw)
|
||||||
|
return self._alt_text(raw)
|
||||||
|
|
||||||
|
def _alt_text(self, raw):
|
||||||
return usertypes.AltText(str(raw), self.type_obj.typename())
|
return usertypes.AltText(str(raw), self.type_obj.typename())
|
||||||
|
|
||||||
def _make_rich_value(self, typed_value):
|
def _make_rich_value(self, typed_value):
|
||||||
@ -440,6 +447,16 @@ class BaseReferenceColumn(BaseColumn):
|
|||||||
"""
|
"""
|
||||||
return self.getdefault()
|
return self.getdefault()
|
||||||
|
|
||||||
|
def _lookup(self, reference_lookup, value):
|
||||||
|
col_id = (
|
||||||
|
reference_lookup.options.get("column")
|
||||||
|
or self._target_table._engine.docmodel
|
||||||
|
.get_column_rec(self.table_id, self.col_id).visibleCol.colId
|
||||||
|
or "id"
|
||||||
|
)
|
||||||
|
target_value = self._target_table.get_column(col_id)._convert_raw_value(value)
|
||||||
|
return self._target_table.lookup_one_record(**{col_id: target_value})
|
||||||
|
|
||||||
|
|
||||||
class ReferenceColumn(BaseReferenceColumn):
|
class ReferenceColumn(BaseReferenceColumn):
|
||||||
"""
|
"""
|
||||||
@ -465,6 +482,11 @@ class ReferenceColumn(BaseReferenceColumn):
|
|||||||
values = action_summary.translate_new_row_ids(self._target_table.table_id, values)
|
values = action_summary.translate_new_row_ids(self._target_table.table_id, values)
|
||||||
return values, []
|
return values, []
|
||||||
|
|
||||||
|
def convert(self, val):
|
||||||
|
if isinstance(val, objtypes.ReferenceLookup):
|
||||||
|
val = self._lookup(val, val.value) or self._alt_text(val.alt_text)
|
||||||
|
return super(ReferenceColumn, self).convert(val)
|
||||||
|
|
||||||
|
|
||||||
class ReferenceListColumn(BaseReferenceColumn):
|
class ReferenceListColumn(BaseReferenceColumn):
|
||||||
"""
|
"""
|
||||||
@ -502,6 +524,21 @@ class ReferenceListColumn(BaseReferenceColumn):
|
|||||||
raw = [r for r in raw if r not in target_row_ids] or None
|
raw = [r for r in raw if r not in target_row_ids] or None
|
||||||
return raw
|
return raw
|
||||||
|
|
||||||
|
def convert(self, val):
|
||||||
|
if isinstance(val, objtypes.ReferenceLookup):
|
||||||
|
result = []
|
||||||
|
values = val.value
|
||||||
|
if not isinstance(values, list):
|
||||||
|
values = [values]
|
||||||
|
for value in values:
|
||||||
|
lookup_value = self._lookup(val, value)
|
||||||
|
if not lookup_value:
|
||||||
|
return self._alt_text(val.alt_text)
|
||||||
|
result.append(lookup_value)
|
||||||
|
val = result
|
||||||
|
return super(ReferenceListColumn, self).convert(val)
|
||||||
|
|
||||||
|
|
||||||
# Set up the relationship between usertypes objects and column objects.
|
# Set up the relationship between usertypes objects and column objects.
|
||||||
usertypes.BaseColumnType.ColType = DataColumn
|
usertypes.BaseColumnType.ColType = DataColumn
|
||||||
usertypes.Reference.ColType = ReferenceColumn
|
usertypes.Reference.ColType = ReferenceColumn
|
||||||
|
@ -232,6 +232,8 @@ def decode_object(value):
|
|||||||
return RaisedException.decode_args(*args)
|
return RaisedException.decode_args(*args)
|
||||||
elif code == 'L':
|
elif code == 'L':
|
||||||
return [decode_object(item) for item in args]
|
return [decode_object(item) for item in args]
|
||||||
|
elif code == 'l':
|
||||||
|
return ReferenceLookup(*args)
|
||||||
elif code == 'O':
|
elif code == 'O':
|
||||||
return {decode_object(key): decode_object(val) for key, val in six.iteritems(args[0])}
|
return {decode_object(key): decode_object(val) for key, val in six.iteritems(args[0])}
|
||||||
elif code == 'P':
|
elif code == 'P':
|
||||||
@ -381,3 +383,19 @@ class RecordSetStub(object):
|
|||||||
def __init__(self, table_id, row_ids):
|
def __init__(self, table_id, row_ids):
|
||||||
self.table_id = table_id
|
self.table_id = table_id
|
||||||
self.row_ids = row_ids
|
self.row_ids = row_ids
|
||||||
|
|
||||||
|
|
||||||
|
class ReferenceLookup(object):
|
||||||
|
def __init__(self, value, options=None):
|
||||||
|
self.value = value
|
||||||
|
self.options = options or {}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def alt_text(self):
|
||||||
|
result = self.options.get("raw")
|
||||||
|
if result is None:
|
||||||
|
values = self.value
|
||||||
|
if not isinstance(values, list):
|
||||||
|
values = [values]
|
||||||
|
result = ", ".join(map(six.text_type, values))
|
||||||
|
return result
|
||||||
|
@ -867,3 +867,95 @@ class TestUserActions(test_engine.EngineTestCase):
|
|||||||
[11, 5],
|
[11, 5],
|
||||||
[12, [[]]],
|
[12, [[]]],
|
||||||
])
|
])
|
||||||
|
|
||||||
|
def test_reference_lookup(self):
|
||||||
|
sample = testutil.parse_test_sample({
|
||||||
|
"SCHEMA": [
|
||||||
|
[1, "Table1", [
|
||||||
|
[1, "name", "Text", False, "", "name", ""],
|
||||||
|
[2, "ref", "Ref:Table1", False, "", "ref", ""],
|
||||||
|
[3, "reflist", "RefList:Table1", False, "", "reflist", ""],
|
||||||
|
]],
|
||||||
|
],
|
||||||
|
"DATA": {
|
||||||
|
"Table1": [
|
||||||
|
["id", "name"],
|
||||||
|
[1, "a"],
|
||||||
|
[2, "b"],
|
||||||
|
],
|
||||||
|
}
|
||||||
|
})
|
||||||
|
self.load_sample(sample)
|
||||||
|
self.update_record("_grist_Tables_column", 2, visibleCol=1)
|
||||||
|
|
||||||
|
# Normal case
|
||||||
|
out_actions = self.apply_user_action(
|
||||||
|
["UpdateRecord", "Table1", 1, {"ref": ["l", "b", {"column": "name"}]}])
|
||||||
|
self.assertPartialOutActions(out_actions, {'stored': [
|
||||||
|
["UpdateRecord", "Table1", 1, {"ref": 2}]]})
|
||||||
|
|
||||||
|
# Use ref.visibleCol (name) as default lookup column
|
||||||
|
out_actions = self.apply_user_action(
|
||||||
|
["UpdateRecord", "Table1", 2, {"ref": ["l", "a"]}])
|
||||||
|
self.assertPartialOutActions(out_actions, {'stored': [
|
||||||
|
["UpdateRecord", "Table1", 2, {"ref": 1}]]})
|
||||||
|
|
||||||
|
# No match found, generate alttext from value
|
||||||
|
out_actions = self.apply_user_action(
|
||||||
|
["UpdateRecord", "Table1", 2, {"ref": ["l", "foo", {"column": "name"}]}])
|
||||||
|
self.assertPartialOutActions(out_actions, {'stored': [
|
||||||
|
["UpdateRecord", "Table1", 2, {"ref": "foo"}]]})
|
||||||
|
|
||||||
|
# No match found, use provided alttext
|
||||||
|
out_actions = self.apply_user_action(
|
||||||
|
["UpdateRecord", "Table1", 2, {"ref": ["l", "foo", {"column": "name", "raw": "alt"}]}])
|
||||||
|
self.assertPartialOutActions(out_actions, {'stored': [
|
||||||
|
["UpdateRecord", "Table1", 2, {"ref": "alt"}]]})
|
||||||
|
|
||||||
|
# Normal case, adding instead of updating
|
||||||
|
out_actions = self.apply_user_action(
|
||||||
|
["AddRecord", "Table1", 3,
|
||||||
|
{"ref": ["l", "b", {"column": "name"}],
|
||||||
|
"name": "c"}])
|
||||||
|
self.assertPartialOutActions(out_actions, {'stored': [
|
||||||
|
["AddRecord", "Table1", 3,
|
||||||
|
{"ref": 2,
|
||||||
|
"name": "c"}]]})
|
||||||
|
|
||||||
|
# Testing reflist and bulk action
|
||||||
|
out_actions = self.apply_user_action(
|
||||||
|
["BulkUpdateRecord", "Table1", [1, 2, 3],
|
||||||
|
{"reflist": [
|
||||||
|
["l", "c", {"column": "name"}], # value gets wrapped in list automatically
|
||||||
|
["l", ["a", "b"], {"column": "name"}], # normal case
|
||||||
|
# "a" matches but "foo" doesn't so the whole thing fails
|
||||||
|
["l", ["a", "foo"], {"column": "name", "raw": "alt"}],
|
||||||
|
]}])
|
||||||
|
self.assertPartialOutActions(out_actions, {'stored': [
|
||||||
|
["BulkUpdateRecord", "Table1", [1, 2, 3],
|
||||||
|
{"reflist": [
|
||||||
|
["L", 3],
|
||||||
|
["L", 1, 2],
|
||||||
|
"alt",
|
||||||
|
]}]]})
|
||||||
|
|
||||||
|
self.assertTableData('Table1', data=[
|
||||||
|
["id", "name", "ref", "reflist"],
|
||||||
|
[1, "a", 2, [3]],
|
||||||
|
[2, "b", "alt", [1, 2]],
|
||||||
|
[3, "c", 2, "alt"],
|
||||||
|
])
|
||||||
|
|
||||||
|
# 'id' is used as the default visibleCol
|
||||||
|
out_actions = self.apply_user_action(
|
||||||
|
["BulkUpdateRecord", "Table1", [1, 2],
|
||||||
|
{"reflist": [
|
||||||
|
["l", 2],
|
||||||
|
["l", 999], # this row ID doesn't exist
|
||||||
|
]}])
|
||||||
|
self.assertPartialOutActions(out_actions, {'stored': [
|
||||||
|
["BulkUpdateRecord", "Table1", [1, 2],
|
||||||
|
{"reflist": [
|
||||||
|
["L", 2],
|
||||||
|
"999",
|
||||||
|
]}]]})
|
||||||
|
Loading…
Reference in New Issue
Block a user