(core) Barely working reference lists in frontend

Summary:
This makes it possible to set the type of a column to ReferenceList, but the UI is terrible

ReferenceList.ts is a mishmash of ChoiceList and Reference that sort of works but something about the CSS is clearly broken

ReferenceListEditor is just a text editor, you have to type in a JSON array of row IDs. Ignore the value that's present when you start editing. I can maybe try mashing together ReferenceEditor and ChoiceListEditor but it doesn't seem wise.
I think @georgegevoian should take over here. Reviewing the diff as it is to check for obvious issues is probably good but I don't think it's worth trying to land/merge anything.

Test Plan: none

Reviewers: dsagal

Reviewed By: dsagal

Subscribers: georgegevoian

Differential Revision: https://phab.getgrist.com/D2914

app/client/components/TypeConversion.ts

@@ -8,6 +8,7 @@ import {DocModel} from 'app/client/models/DocModel';
 import {ColumnRec} from 'app/client/models/entities/ColumnRec';
 import * as UserType from 'app/client/widgets/UserType';
 import * as gristTypes from 'app/common/gristTypes';
+import {isFullReferencingType} from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
 import {TableData} from 'app/common/TableData';
 
@@ -26,9 +27,11 @@ export interface ColInfo {
  */
 export function addColTypeSuffix(type: string, column: ColumnRec, docModel: DocModel) {
   switch (type) {
-    case "Ref": {
+    case "Ref":
+    case "RefList":
+    {
       const refTableId = getRefTableIdFromData(docModel, column) || column.table().primaryTableId();
-      return 'Ref:' + refTableId;
+      return `${type}:${refTableId}`;
     }
     case "DateTime":
       return 'DateTime:' + docModel.docInfo.getRowModel(1).timezone();
@@ -118,10 +121,12 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
       }
       break;
     }
-    case 'Ref': {
+    case 'Ref':
+    case 'RefList':
+    {
       // Set suggested destination table and visible column.
       // Null if toTypeMaybeFull is a pure type (e.g. converting to Ref before a table is chosen).
-      const optTableId = gutil.removePrefix(toTypeMaybeFull, "Ref:")!;
+      const optTableId = gutil.removePrefix(toTypeMaybeFull, `${toType}:`)!;
 
       // Finds a reference suggestion column and sets it as the current reference value.
       const columnData = tableData.getDistinctValues(origDisplayCol.colId(), 100);
@@ -138,7 +143,7 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
         console.warn("Inappropriate column received from findColFromValues");
         break;
       }
-      colInfo.type = `Ref:${suggestedTableId}`;
+      colInfo.type = `${toType}:${suggestedTableId}`;
       colInfo.visibleCol = suggestedColRef;
       break;
     }
@@ -177,8 +182,10 @@ export function getDefaultFormula(
 
   let origValFormula = oldVisibleColName ?
     // The `str()` below converts AltText to plain text.
-    `$${colId}.${oldVisibleColName} if ISREF($${colId}) else str($${colId})` :
+    `($${colId}.${oldVisibleColName}
-    `$${colId}`;
+    if ISREF($${colId}) or ISREFLIST($${colId})
+    else str($${colId}))`
+    : `$${colId}`;
 
   if (origCol.type.peek() === 'ChoiceList') {
     origValFormula = `grist.ChoiceList.toString($${colId})`;
@@ -190,8 +197,10 @@ export function getDefaultFormula(
   // Optional parameters depend on the type; see sandbox/grist/usertypes.py
   const args: string[] = [origValFormula];
   switch (toTypePure) {
-    case 'Ref': {
+    case 'Ref':
-      const table = gutil.removePrefix(newType, "Ref:");
+    case 'RefList':
+    {
+      const table = gutil.removePrefix(newType, toTypePure + ":");
       args.push(table || 'None');
       const visibleColName = getVisibleColName(docModel, newVisibleCol);
       if (visibleColName) {
@@ -222,7 +231,7 @@ function getVisibleColName(docModel: DocModel, visibleColRef: number): string|un
   return visibleColRef ? docModel.columns.getRowModel(visibleColRef).colId() : undefined;
 }
 
-// Returns whether the given column model is of type Ref.
+// Returns whether the given column model is of type Ref or RefList.
 function isReferenceCol(colModel: ColumnRec) {
-  return gristTypes.extractTypeFromColType(colModel.type()) === 'Ref';
+  return isFullReferencingType(colModel.type());
 }

app/client/models/entities/ColumnRec.ts

@@ -2,7 +2,7 @@ import {KoArray} from 'app/client/lib/koArray';
 import {DocModel, IRowModel, recordSet, refRecord, TableRec, ViewFieldRec} from 'app/client/models/DocModel';
 import {jsonObservable, ObjObservable} from 'app/client/models/modelUtil';
 import * as gristTypes from 'app/common/gristTypes';
-import {removePrefix} from 'app/common/gutil';
+import {getReferencedTableId} from 'app/common/gristTypes';
 import * as ko from 'knockout';
 
 // Represents a column in a user-defined table.
@@ -87,7 +87,7 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void {
 
   // Returns the rowModel for the referenced table, or null, if this is not a reference column.
   this.refTable = ko.pureComputed(() => {
-    const refTableId = removePrefix(this.type() || "", 'Ref:');
+    const refTableId = getReferencedTableId(this.type() || "");
     return refTableId ? docModel.allTables.all().find(t => t.tableId() === refTableId) || null : null;
   });
 }

app/client/widgets/ChoiceListCell.ts

@@ -49,7 +49,7 @@ export class ChoiceListCell extends ChoiceTextBox {
   }
 }
 
-const cssChoiceList = styled('div', `
+export const cssChoiceList = styled('div', `
   display: flex;
   align-items: start;
   padding: 0 3px;
@@ -63,7 +63,7 @@ const cssChoiceList = styled('div', `
   }
 `);
 
-const cssToken = styled('div', `
+export const cssToken = styled('div', `
   flex: 0 1 auto;
   min-width: 0px;
   margin: 2px;

app/client/widgets/FieldBuilder.ts

@@ -25,7 +25,7 @@ import { NewBaseEditor } from "app/client/widgets/NewBaseEditor";
 import * as UserType from 'app/client/widgets/UserType';
 import * as UserTypeImpl from 'app/client/widgets/UserTypeImpl';
 import * as gristTypes from 'app/common/gristTypes';
-import * as gutil from 'app/common/gutil';
+import { getReferencedTableId, isFullReferencingType } from 'app/common/gristTypes';
 import { CellValue } from 'app/plugin/GristData';
 import { Computed, Disposable, fromKo, dom as grainjsDom,
          Holder, IDisposable, makeTestId, toKo } from 'grainjs';
@@ -115,15 +115,22 @@ export class FieldBuilder extends Disposable {
       return gristTypes.isRightType(this._readOnlyPureType()) || _.constant(false);
     }, this);
 
-    // Returns a boolean indicating whether the column is type Reference.
+    // Returns a boolean indicating whether the column is type Reference or ReferenceList.
     this._isRef = this.autoDispose(ko.computed(() => {
-      return gutil.startsWith(this.field.column().type(), 'Ref:');
+      return isFullReferencingType(this.field.column().type());
     }));
 
     // Gives the table ID to which the reference points.
     this._refTableId = this.autoDispose(ko.computed({
-      read: () => gutil.removePrefix(this.field.column().type(), "Ref:"),
+      read: () => getReferencedTableId(this.field.column().type()),
-      write: val => this._setType(`Ref:${val}`)
+      write: val => {
+        const type = this.field.column().type();
+        if (type.startsWith('Ref:')) {
+          void this._setType(`Ref:${val}`);
+        } else {
+          void this._setType(`RefList:${val}`);
+        }
+      }
     }));
 
     this.widget = ko.pureComputed({

app/client/widgets/Reference.ts

@@ -5,7 +5,7 @@ import {colors, testId} from 'app/client/ui2018/cssVars';
 import {icon} from 'app/client/ui2018/icons';
 import {IOptionFull, select} from 'app/client/ui2018/menus';
 import {NTextBox} from 'app/client/widgets/NTextBox';
-import {isVersions} from 'app/common/gristTypes';
+import {isFullReferencingType, isVersions} from 'app/common/gristTypes';
 import {BaseFormatter} from 'app/common/ValueFormatter';
 import {Computed, dom, styled} from 'grainjs';
 import * as ko from 'knockout';
@@ -14,10 +14,11 @@ import * as ko from 'knockout';
  * Reference - The widget for displaying references to another table's records.
  */
 export class Reference extends NTextBox {
+  protected _formatValue: Computed<(val: any) => string>;
+
   private _refValueFormatter: ko.Computed<BaseFormatter>;
   private _visibleColRef: Computed<number>;
   private _validCols: Computed<Array<IOptionFull<number>>>;
-  private _formatValue: Computed<(val: any) => string>;
 
   constructor(field: ViewFieldRec) {
     super(field);
@@ -38,7 +39,7 @@ export class Reference extends NTextBox {
           label: use(col.label),
           value: col.getRowId(),
           icon: 'FieldColumn',
-          disabled: use(col.type).startsWith('Ref:') || use(col.isTransforming)
+          disabled: isFullReferencingType(use(col.type)) || use(col.isTransforming)
         }))
         .concat([{label: 'Row ID', value: 0, icon: 'FieldColumn'}]);
     });

app/client/widgets/ReferenceList.ts

@@ -0,0 +1,54 @@
+import {DataRowModel} from 'app/client/models/DataRowModel';
+import {testId} from 'app/client/ui2018/cssVars';
+import {isList} from 'app/common/gristTypes';
+import {dom} from 'grainjs';
+import {cssChoiceList, cssToken} from "./ChoiceListCell";
+import {Reference} from "./Reference";
+import {choiceToken} from "./ChoiceToken";
+
+/**
+ * ReferenceList - The widget for displaying lists of references to another table's records.
+ */
+export class ReferenceList extends Reference {
+  public buildDom(row: DataRowModel) {
+    return cssChoiceList(
+      dom.cls('field_clip'),
+      cssChoiceList.cls('-wrap', this.wrapping),
+      dom.style('justify-content', use => use(this.alignment) === 'right' ? 'flex-end' : use(this.alignment)),
+      dom.domComputed((use) => {
+
+        if (use(row._isAddRow) || this.isDisposed() || use(this.field.displayColModel).isDisposed()) {
+          // Work around JS errors during certain changes (noticed when visibleCol field gets removed
+          // for a column using per-field settings).
+          return null;
+        }
+        const value = row.cells[use(use(this.field.displayColModel).colId)];
+        if (!value) {
+          return null;
+        }
+        const content = use(value);
+        // if (isVersions(content)) {  // TODO
+        //   // We can arrive here if the reference value is unchanged (viewed as a foreign key)
+        //   // but the content of its displayCol has changed.  Postponing doing anything about
+        //   // this until we have three-way information for computed columns.  For now,
+        //   // just showing one version of the cell.  TODO: elaborate.
+        //   return use(this._formatValue)(content[1].local || content[1].parent);
+        // }
+        const items = isList(content) ? content.slice(1) : [content];
+        return items.map(use(this._formatValue));
+      }, (input) => {
+        if (!input) {
+          return null;
+        }
+        return input.map(token =>
+          choiceToken(
+            String(token),
+            {},  // default colors
+            dom.cls(cssToken.className),
+            testId('ref-list-cell-token')
+          ),
+        );
+      }),
+    );
+  }
+}

app/client/widgets/ReferenceListEditor.ts

@@ -0,0 +1,16 @@
+import {NTextEditor} from 'app/client/widgets/NTextEditor';
+import {CellValue} from "app/common/DocActions";
+
+
+/**
+ * A ReferenceListEditor offers an autocomplete of choices from the referenced table.
+ */
+export class ReferenceListEditor extends NTextEditor {
+  public getCellValue(): CellValue {
+    try {
+      return ['L', ...JSON.parse(this.textInput.value)];
+    } catch {
+      return null;   // This is the default value for a reference list column.
+    }
+  }
+}

app/client/widgets/UserType.js

@@ -237,6 +237,21 @@ var typeDefs = {
     },
     default: 'Reference'
   },
+  // RefList: {
+  //   label: 'Reference List',
+  //   icon: 'FieldReference',
+  //   widgets: {
+  //     Reference: {
+  //       cons: 'ReferenceList',
+  //       editCons: 'ReferenceListEditor',
+  //       icon: 'FieldReference',
+  //       options: {
+  //         alignment: 'left'
+  //       }
+  //     }
+  //   },
+  //   default: 'Reference'
+  // },
   Attachments: {
     label: 'Attachment',
     icon: 'FieldAttachment',

app/client/widgets/UserTypeImpl.js

@@ -7,6 +7,8 @@ const UserType = require('./UserType');
 const {HyperLinkEditor} = require('./HyperLinkEditor');
 const {NTextEditor} = require('./NTextEditor');
 const {ReferenceEditor} = require('./ReferenceEditor');
+const {ReferenceList} = require('./ReferenceList');
+const {ReferenceListEditor} = require('./ReferenceListEditor');
 const {HyperLinkTextBox} = require('./HyperLinkTextBox');
 const {ChoiceTextBox } = require('./ChoiceTextBox');
 const {Reference} = require('./Reference');
@@ -26,6 +28,8 @@ const nameToWidget = {
   'Reference': Reference,
   'Switch': require('./Switch'),
   'ReferenceEditor': ReferenceEditor,
+  'ReferenceList': ReferenceList,
+  'ReferenceListEditor': ReferenceListEditor,
   'ChoiceTextBox': ChoiceTextBox,
   'ChoiceEditor': require('./ChoiceEditor'),
   'ChoiceListCell': require('./ChoiceListCell').ChoiceListCell,

app/common/gristTypes.ts

@@ -1,5 +1,6 @@
 import { CellValue, CellVersions } from 'app/common/DocActions';
 import isString = require('lodash/isString');
+import {removePrefix} from "./gutil";
 
 // tslint:disable:object-literal-key-quotes
 
@@ -10,7 +11,8 @@ export type GristType = 'Any' | 'Attachments' | 'Blob' | 'Bool' | 'Choice' | 'Ch
 export type GristTypeInfo =
   {type: 'DateTime', timezone: string} |
   {type: 'Ref', tableId: string} |
-  {type: Exclude<GristType, 'DateTime'|'Ref'>};
+  {type: 'RefList', tableId: string} |
+  {type: Exclude<GristType, 'DateTime'|'Ref'|'RefList'>};
 
 
 // Letter codes for CellValue types encoded as [code, args...] tuples.
@@ -76,6 +78,7 @@ export function extractInfoFromColType(colType: string): GristTypeInfo {
   const colon = colType.indexOf(':');
   const [type, arg] = (colon === -1) ? [colType] : [colType.slice(0, colon), colType.slice(colon + 1)];
   return (type === 'Ref') ? {type, tableId: String(arg)} :
+    (type === 'RefList')  ? {type, tableId: String(arg)} :
     (type === 'DateTime') ? {type, timezone: String(arg)} :
     {type} as GristTypeInfo;
 }
@@ -321,3 +324,11 @@ export function sequelizeToGristType(sqlType: string): GristType {
       throw new Error('Unrecognized datatype: `' + sqlType + '`');
   }
 }
+
+export function getReferencedTableId(type: string) {
+  return removePrefix(type, "Ref:") || removePrefix(type, "RefList:");
+}
+
+export function isFullReferencingType(type: string) {
+  return type.startsWith('Ref:') || type.startsWith('RefList:');
+}

app/server/lib/DocStorage.ts

@@ -26,6 +26,7 @@ import * as _ from 'underscore';
 import * as util from 'util';
 import * as uuidv4 from "uuid/v4";
 import { ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB';
+import {isList} from "app/common/gristTypes";
 
 
 // Run with environment variable NODE_DEBUG=db (may include additional comma-separated sections)
@@ -449,7 +450,11 @@ export class DocStorage implements ISQLiteDB {
     if (gristType == 'ChoiceList') {
       // See also app/plugin/objtype.ts for decodeObject(). Here we manually check and decode
       // the "List" object type.
-      if (Array.isArray(val) && val[0] === 'L' && val.every(tok => (typeof(tok) === 'string'))) {
+      if (isList(val) && val.every(tok => (typeof(tok) === 'string'))) {
+        return JSON.stringify(val.slice(1));
+      }
+    } else if (gristType?.startsWith('RefList:')) {
+      if (isList(val) && val.slice(1).every((tok: any) => (typeof(tok) === 'number'))) {
         return JSON.stringify(val.slice(1));
       }
     }
@@ -515,7 +520,7 @@ export class DocStorage implements ISQLiteDB {
         return Boolean(val);
       }
     }
-    if (gristType === 'ChoiceList') {
+    if (gristType === 'ChoiceList' || gristType?.startsWith('RefList:')) {
       if (typeof val === 'string' && val.startsWith('[')) {
         try {
           return ['L', ...JSON.parse(val)];
@@ -557,6 +562,8 @@ export class DocStorage implements ISQLiteDB {
       case 'Text':
         return 'TEXT';
       case 'ChoiceList':
+      case 'RefList':
+      case 'ReferenceList':
         return 'TEXT';      // To be encoded as a JSON array of strings.
       case 'Date':
         return 'DATE';
@@ -572,7 +579,14 @@ export class DocStorage implements ISQLiteDB {
       case 'PositionNumber':
         return 'NUMERIC';
     }
-    if (colType && colType.startsWith('Ref:')) { return 'INTEGER'; }
+    if (colType) {
+      if (colType.startsWith('Ref:')) {
+        return 'INTEGER';
+      }
+      if (colType.startsWith('RefList:')) {
+        return 'TEXT';      // To be encoded as a JSON array of strings.
+      }
+    }
     return 'BLOB';
   }
 

sandbox/grist/column.py

@@ -440,6 +440,14 @@ class ReferenceListColumn(BaseReferenceColumn):
   ReferenceListColumn maintains for each row a list of references (row IDs) into another table.
   Accessing them yields RecordSets.
   """
+  def set(self, row_id, value):
+    if isinstance(value, six.string_types) and value.startswith(u'['):
+      try:
+        value = json.loads(value)
+      except Exception:
+        pass
+    super(ReferenceListColumn, self).set(row_id, value)
+
   def _update_references(self, row_id, old_list, new_list):
     for old_value in old_list or ():
       self._relation.remove_reference(row_id, old_value)

sandbox/grist/functions/info.py

@@ -188,10 +188,10 @@ def ISREF(value):
   """
   Checks whether a value is a table record.
 
-  For example, if a column person is of type Reference to the People table, then ISREF($person)
+  For example, if a column `person` is of type Reference to the `People` table,
-  is True.
+  then `ISREF($person)` is `True`.
-  Similarly, ISREF(People.lookupOne(name=$name)) is True. For any other type of value,
+  Similarly, `ISREF(People.lookupOne(name=$name))` is `True`. For any other type of value,
-  ISREF() would evaluate to False.
+  `ISREF()` would evaluate to `False`.
 
   >>> ISREF(17)
   False
@@ -202,6 +202,25 @@ def ISREF(value):
   return isinstance(value, Record)
 
 
+def ISREFLIST(value):
+  """
+  Checks whether a value is a [`RecordSet`](#recordset),
+  the type of values in Reference List columns.
+
+  For example, if a column `people` is of type Reference List to the `People` table,
+  then `ISREFLIST($people)` is `True`.
+  Similarly, `ISREFLIST(People.lookupRecords(name=$name))` is `True`. For any other type of value,
+  `ISREFLIST()` would evaluate to `False`.
+
+  >>> ISREFLIST(17)
+  False
+  >>> ISREFLIST("Roger")
+  False
+
+  """
+  return isinstance(value, RecordSet)
+
+
 def ISTEXT(value):
   """
   Checks whether a value is text.

sandbox/grist/objtypes.py

@@ -286,7 +286,7 @@ class RecordList(list):
     self._sort_by = sort_by
 
   def __repr__(self):
-    return "RecordList(%r, group_by=%r, sort_by=%r)" % (
+    return "RecordList(%s, group_by=%r, sort_by=%r)" % (
       list.__repr__(self), self._group_by, self._sort_by)
 
 

sandbox/grist/summary.py

@@ -121,11 +121,16 @@ class SummaryActions(object):
     """
     key = tuple(sorted(int(c) for c in source_groupby_columns))
 
-    groupby_colinfo = [_make_col_info(col=c,
+    groupby_colinfo = [
-                                      isFormula=False,
+      _make_col_info(
-                                      formula='',
+        col=c,
-                                      type='Choice' if c.type == 'ChoiceList' else c.type)
+        isFormula=False,
-                       for c in source_groupby_columns]
+        formula='',
+        type='Choice' if c.type == 'ChoiceList' else
+        c.type.replace('RefList:', 'Ref:')
+      )
+      for c in source_groupby_columns
+    ]
     summary_table = next((t for t in source_table.summaryTables if t.summaryKey == key), None)
     created = False
     if not summary_table:

sandbox/grist/table.py

@@ -270,7 +270,7 @@ class Table(object):
       self._summary_simple = not any(
         isinstance(
           self._summary_source_table.all_columns.get(group_col),
-          column.ChoiceListColumn
+          (column.ChoiceListColumn, column.ReferenceListColumn)
         )
         for group_col in groupby_cols
       )
@@ -299,12 +299,13 @@ class Table(object):
       @usertypes.formulaType(usertypes.ReferenceList(summary_table.table_id))
       def _updateSummary(rec, table):  # pylint: disable=unused-argument
         # Create a row in the summary table for every combination of values in
-        # ChoiceList columns
+        # list type columns
         lookup_values = []
         for group_col in groupby_cols:
           lookup_value = getattr(rec, group_col)
-          if isinstance(self.all_columns[group_col], column.ChoiceListColumn):
+          if isinstance(self.all_columns[group_col],
-            # Check that ChoiceList cells have appropriate types.
+                        (column.ChoiceListColumn, column.ReferenceListColumn)):
+            # Check that ChoiceList/ReferenceList cells have appropriate types.
             # Don't iterate over characters of a string.
             if isinstance(lookup_value, (six.binary_type, six.text_type)):
               return []

sandbox/grist/usertypes.py

@@ -453,6 +453,14 @@ class ReferenceList(BaseColumnType):
     return "RefList"
 
   def do_convert(self, value):
+    if isinstance(value, six.string_types):
+      # If it's a string that looks like JSON, try to parse it as such.
+      if value.startswith('['):
+        try:
+          value = json.loads(value)
+        except Exception:
+          pass
+
     if isinstance(value, RecordSet):
       assert value._table.table_id == self.table_id
       return objtypes.RecordList(value._row_ids, group_by=value._group_by, sort_by=value._sort_by)
@@ -466,9 +474,24 @@ class ReferenceList(BaseColumnType):
     return value is None or (isinstance(value, list) and
                              all(Reference.is_right_type(val) for val in value))
 
+  @classmethod
+  def typeConvert(cls, value, ref_table, visible_col=None):  # noqa # pylint: disable=arguments-differ
+    # TODO this is based on Reference.typeConvert.
+    #  It doesn't make much sense as a conversion but I don't know what would
+    if ref_table and visible_col:
+      return ref_table.lookupRecords(**{visible_col: value}) or six.text_type(value)
+    else:
+      return value
+
+
 class Attachments(ReferenceList):
   """
   Currently attachment type is the field for holding data for attachments.
   """
   def __init__(self):
     super(Attachments, self).__init__('_grist_Attachments')
+
+  @classmethod
+  def typeConvert(cls, value):  # noqa # pylint: disable=arguments-differ
+    # Don't use ReferenceList.typeConvert which is called with a different number of arguments
+    return value