(core) Adding sort options for columns.

Summary:
Adding sort options for columns.
- Sort menu has a new option "More sort options" that opens up Sort left menu
- Each sort entry has an additional menu with 3 options
-- Order by choice index (for the Choice column, orders by choice position)
-- Empty last (puts empty values last in ascending order, first in descending order)
-- Natural sort (for Text column, compares strings with numbers as numbers)
Updated also CSV/Excel export and api sorting.
Most of the changes in this diff is a sort expression refactoring. Pulling out all the methods
that works on sortExpression array into a single namespace.

Test Plan: Browser tests

Reviewers: alexmojaki

Reviewed By: alexmojaki

Subscribers: dsagal, alexmojaki

Differential Revision: https://phab.getgrist.com/D3077

app/server/lib/ActiveDoc.ts

@@ -842,7 +842,10 @@ export class ActiveDoc extends EventEmitter {
    * @returns {Promise<TableRecordValue[]>} Records containing metadata about the visible columns
    * from `tableId`.
    */
-  public async getTableCols(docSession: OptDocSession, tableId: string): Promise<TableRecordValue[]> {
+  public async getTableCols(
+    docSession: OptDocSession,
+    tableId: string,
+    includeHidden = false): Promise<TableRecordValue[]> {
     const metaTables = await this.fetchMetaTables(docSession);
     const tableRef = tableIdToRef(metaTables, tableId);
     const [, , colRefs, columnData] = metaTables._grist_Tables_column;
@@ -852,12 +855,11 @@ export class ActiveDoc extends EventEmitter {
 
     const columns: TableRecordValue[] = [];
     (columnData.colId as string[]).forEach((id, index) => {
-      if (
-        // TODO param to include hidden columns
-        id === "manualSort" || id.startsWith("gristHelper_") || !id ||
-        // Filter columns from the requested table
-        columnData.parentId[index] !== tableRef
-      ) {
+      const hasNoId = !id;
+      const isHidden = hasNoId || id === "manualSort" || id.startsWith("gristHelper_");
+      const fromDifferentTable = columnData.parentId[index] !== tableRef;
+      const skip = (isHidden && !includeHidden) || hasNoId || fromDifferentTable;
+      if (skip) {
         return;
       }
       const column: TableRecordValue = { id, fields: { colRef: colRefs[index] } };

app/server/lib/DocApi.ts

@@ -2,7 +2,7 @@ import { createEmptyActionSummary } from "app/common/ActionSummary";
 import { ApiError } from 'app/common/ApiError';
 import { BrowserSettings } from "app/common/BrowserSettings";
 import {
-  BulkColValues, CellValue, fromTableDataAction, TableColValues, TableRecordValue,
+  BulkColValues, CellValue, ColValues, fromTableDataAction, TableColValues, TableRecordValue,
 } from 'app/common/DocActions';
 import {isRaisedException} from "app/common/gristTypes";
 import { arrayRepeat, isAffirmative } from "app/common/gutil";
@@ -45,6 +45,8 @@ import * as path from 'path';
 import * as uuidv4 from "uuid/v4";
 import * as t from "ts-interface-checker";
 import { Checker } from "ts-interface-checker";
+import { ServerColumnGetters } from 'app/server/lib/ServerColumnGetters';
+import { Sort } from 'app/common/SortSpec';
 
 // Cap on the number of requests that can be outstanding on a single document via the
 // rest doc api.  When this limit is exceeded, incoming requests receive an immediate
@@ -153,12 +155,17 @@ export class DocWorkerApi {
         throw new ApiError("Invalid query: filter values must be arrays", 400);
       }
       const tableId = req.params.tableId;
+      const session = docSessionFromRequest(req);
       const tableData = await handleSandboxError(tableId, [], activeDoc.fetchQuery(
-        docSessionFromRequest(req), {tableId, filters}, !immediate));
+        session, {tableId, filters}, !immediate));
+      // For metaTables we don't need to specify columns, search will infer it from the sort expression.
+      const isMetaTable = tableId.startsWith('_grist');
+      const columns = isMetaTable ? null :
+        await handleSandboxError('', [], activeDoc.getTableCols(session, tableId, true));
+      const params = getQueryParameters(req);
       // Apply sort/limit parameters, if set.  TODO: move sorting/limiting into data engine
       // and sql.
-      const params = getQueryParameters(req);
-      return applyQueryParameters(fromTableDataAction(tableData), params);
+      return applyQueryParameters(fromTableDataAction(tableData), params, columns);
     }
 
     // Get the specified table in column-oriented format
@@ -945,8 +952,9 @@ async function handleSandboxError<T>(tableId: string, colNames: string[], p: Pro
  * results returned to the user.
  */
 export interface QueryParameters {
-  sort?: string[];  // Columns to sort by (ascending order by default,
-                    // prepend "-" for descending order).
+  sort?: string[];  // Columns names to sort by (ascending order by default,
+                    // prepend "-" for descending order, can contain flags,
+                    // see more in Sort.SortSpec).
   limit?: number;   // Limit on number of rows to return.
 }
 
@@ -992,29 +1000,51 @@ function getQueryParameters(req: Request): QueryParameters {
 /**
  * Sort table contents being returned.  Sort keys with a '-' prefix
  * are sorted in descending order, otherwise ascending.  Contents are
- * modified in place.
+ * modified in place. Sort keys can contain sort options.
+ * Columns can be either expressed as a colId (name string) or as colRef (rowId number).
  */
-function applySort(values: TableColValues, sort: string[]) {
+function applySort(
+  values: TableColValues,
+  sort: string[],
+  _columns: TableRecordValue[]|null = null) {
   if (!sort) { return values; }
-  const sortKeys = sort.map(key => key.replace(/^-/, ''));
-  const iteratees = sortKeys.map(key => {
-    if (!(key in values)) {
-      throw new Error(`unknown key ${key}`);
-    }
-    const col = values[key];
-    return (i: number) => col[i];
-  });
-  const sortSpec = sort.map((key, i) => (key.startsWith('-') ? -i - 1 : i + 1));
-  const index = values.id.map((_, i) => i);
-  const sortFunc = new SortFunc({
-    getColGetter(i) { return iteratees[i - 1]; },
-    getManualSortGetter() { return null; }
-  });
-  sortFunc.updateSpec(sortSpec);
-  index.sort(sortFunc.compare.bind(sortFunc));
+
+  // First we need to prepare column description in ColValue format (plain objects).
+  // This format is used by ServerColumnGetters.
+  let properColumns: ColValues[] = [];
+
+  // We will receive columns information only for user tables, not for metatables. So
+  // if this is the case, we will infer them from the result.
+  if (!_columns) {
+    _columns = Object.keys(values).map((col, index) => ({ id: col, fields: { colRef: index }}));
+  }
+  // For user tables, we will not get id column (as this column is not in the schema), so we need to
+  // make sure the column is there.
+  else {
+    // This is enough information for ServerGetters
+    _columns = [..._columns, { id : 'id', fields: {colRef: 0 }}];
+  }
+
+  // Once we have proper columns, we can convert them to format that ServerColumnGetters
+  // understand.
+  properColumns = _columns.map(c => ({
+    ...c.fields,
+    id : c.fields.colRef,
+    colId: c.id
+  }));
+
+  // We will sort row indices in the values object, not rows ids.
+  const rowIndices = values.id.map((__, i) => i);
+  const getters = new ServerColumnGetters(rowIndices, values, properColumns);
+  const sortFunc = new SortFunc(getters);
+  const colIdToRef = new Map(properColumns.map(({id, colId}) => [colId as string, id as number]));
+  sortFunc.updateSpec(Sort.parseNames(sort, colIdToRef));
+  rowIndices.sort(sortFunc.compare.bind(sortFunc));
+
+  // Sort resulting values according to the sorted index.
   for (const key of Object.keys(values)) {
     const col = values[key];
-    values[key] = index.map(i => col[i]);
+    values[key] = rowIndices.map(i => col[i]);
   }
   return values;
 }
@@ -1034,8 +1064,11 @@ function applyLimit(values: TableColValues, limit: number) {
 /**
  * Apply query parameters to table contents.  Contents are modified in place.
  */
-export function applyQueryParameters(values: TableColValues, params: QueryParameters): TableColValues {
-  if (params.sort) { applySort(values, params.sort); }
+export function applyQueryParameters(
+  values: TableColValues,
+  params: QueryParameters,
+  columns: TableRecordValue[]|null = null): TableColValues {
+  if (params.sort) { applySort(values, params.sort, columns); }
   if (params.limit) { applyLimit(values, params.limit); }
   return values;
 }

app/server/lib/Export.ts

@@ -1,18 +1,19 @@
-import {buildColFilter} from 'app/common/ColumnFilterFunc';
-import {RowRecord} from 'app/common/DocActions';
-import {DocData} from 'app/common/DocData';
+import { buildColFilter } from 'app/common/ColumnFilterFunc';
+import { RowRecord } from 'app/common/DocActions';
+import { DocData } from 'app/common/DocData';
+import { DocumentSettings } from 'app/common/DocumentSettings';
 import * as gristTypes from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
-import {buildRowFilter} from 'app/common/RowFilterFunc';
-import {SchemaTypes} from 'app/common/schema';
-import {SortFunc} from 'app/common/SortFunc';
-import {TableData} from 'app/common/TableData';
-import {DocumentSettings} from 'app/common/DocumentSettings';
-import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {RequestWithLogin} from 'app/server/lib/Authorizer';
-import {docSessionFromRequest} from 'app/server/lib/DocSession';
-import {optIntegerParam, optJsonParam, stringParam} from 'app/server/lib/requestUtils';
-import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters';
+import { buildRowFilter } from 'app/common/RowFilterFunc';
+import { SchemaTypes } from 'app/common/schema';
+import { SortFunc } from 'app/common/SortFunc';
+import { Sort } from 'app/common/SortSpec';
+import { TableData } from 'app/common/TableData';
+import { ActiveDoc } from 'app/server/lib/ActiveDoc';
+import { RequestWithLogin } from 'app/server/lib/Authorizer';
+import { docSessionFromRequest } from 'app/server/lib/DocSession';
+import { optIntegerParam, optJsonParam, stringParam } from 'app/server/lib/requestUtils';
+import { ServerColumnGetters } from 'app/server/lib/ServerColumnGetters';
 import * as express from 'express';
 import * as _ from 'underscore';
 
@@ -203,7 +204,7 @@ export async function exportTable(
 export async function exportSection(
   activeDoc: ActiveDoc,
   viewSectionId: number,
-  sortOrder: number[] | null,
+  sortSpec: Sort.SortSpec | null,
   filters: Filter[] | null,
   req: express.Request): Promise<ExportData> {
 
@@ -241,16 +242,16 @@ export async function exportSection(
     (field) => viewify(tableColsById[field.colRef], field));
 
   // The columns named in sort order need to now become display columns
-  sortOrder = sortOrder || gutil.safeJsonParse(viewSection.sortColRefs, []);
+  sortSpec = sortSpec || gutil.safeJsonParse(viewSection.sortColRefs, []);
   const fieldsByColRef = _.indexBy(fields, 'colRef');
-  sortOrder = sortOrder!.map((directionalColRef) => {
-    const colRef = Math.abs(directionalColRef);
+  sortSpec = sortSpec!.map((colSpec) => {
+    const colRef = Sort.getColRef(colSpec);
     const col = tableColsById[colRef];
     if (!col) {
       return 0;
     }
     const effectiveColRef = viewify(col, fieldsByColRef[colRef]).id;
-    return directionalColRef > 0 ? effectiveColRef : -effectiveColRef;
+    return Sort.swapColRef(colSpec, effectiveColRef);
   });
 
   // fetch actual data
@@ -260,7 +261,7 @@ export async function exportSection(
   // sort rows
   const getters = new ServerColumnGetters(rowIds, dataByColId, columns);
   const sorter = new SortFunc(getters);
-  sorter.updateSpec(sortOrder);
+  sorter.updateSpec(sortSpec);
   rowIds.sort((a, b) => sorter.compare(a, b));
   // create cell accessors
   const access = viewColumns.map(col => getters.getColGetter(col.id)!);

app/server/lib/ServerColumnGetters.ts

@@ -1,5 +1,8 @@
-import {ColumnGetters} from 'app/common/ColumnGetters';
+import { ColumnGetter, ColumnGetters } from 'app/common/ColumnGetters';
 import * as gristTypes from 'app/common/gristTypes';
+import { safeJsonParse } from 'app/common/gutil';
+import { choiceGetter } from 'app/common/SortFunc';
+import { Sort } from 'app/common/SortSpec';
 
 /**
  *
@@ -12,23 +15,33 @@ export class ServerColumnGetters implements ColumnGetters {
   private _colIndices: Map<number, string>;
 
   constructor(rowIds: number[], private _dataByColId: {[colId: string]: any}, private _columns: any[]) {
-    this._rowIndices = new Map<number, number>(rowIds.map((rowId, r) => [rowId, r] as [number, number]));
+    this._rowIndices = new Map<number, number>(rowIds.map((rowId, index) => [rowId, index] as [number, number]));
     this._colIndices = new Map<number, string>(_columns.map(col => [col.id, col.colId] as [number, string]));
   }
 
-  public getColGetter(colRef: number): ((rowId: number) => any) | null {
+  public getColGetter(colSpec: Sort.ColSpec): ColumnGetter | null {
+    const colRef = Sort.getColRef(colSpec);
     const colId = this._colIndices.get(colRef);
     if (colId === undefined) {
       return null;
     }
     const col = this._dataByColId[colId];
-    return rowId => {
+    let getter = (rowId: number) => {
       const idx = this._rowIndices.get(rowId);
       if (idx === undefined) {
         return null;
       }
       return col[idx];
     };
+    const details = Sort.specToDetails(colSpec);
+    if (details.orderByChoice) {
+      const rowModel = this._columns.find(c => c.id == colRef);
+      if (rowModel?.type === 'Choice') {
+        const choices: string[] = safeJsonParse(rowModel.widgetOptions, {}).choices || [];
+        getter = choiceGetter(getter, choices);
+      }
+    }
+    return getter;
   }
 
   public getManualSortGetter(): ((rowId: number) => any) | null {