(core) Adding sort options for columns.

Summary:
Adding sort options for columns.
- Sort menu has a new option "More sort options" that opens up Sort left menu
- Each sort entry has an additional menu with 3 options
-- Order by choice index (for the Choice column, orders by choice position)
-- Empty last (puts empty values last in ascending order, first in descending order)
-- Natural sort (for Text column, compares strings with numbers as numbers)
Updated also CSV/Excel export and api sorting.
Most of the changes in this diff is a sort expression refactoring. Pulling out all the methods
that works on sortExpression array into a single namespace.

Test Plan: Browser tests

Reviewers: alexmojaki

Reviewed By: alexmojaki

Subscribers: dsagal, alexmojaki

Differential Revision: https://phab.getgrist.com/D3077

app/client/components/GridView.js

@@ -6,13 +6,14 @@ var ko        = require('knockout');
 var gutil             = require('app/common/gutil');
 var BinaryIndexedTree = require('app/common/BinaryIndexedTree');
 var MANUALSORT        = require('app/common/gristTypes').MANUALSORT;
+const {Sort} = require('app/common/SortSpec');
 
 var dom           = require('../lib/dom');
 var kd            = require('../lib/koDom');
 var kf            = require('../lib/koForm');
 var koDomScrolly  = require('../lib/koDomScrolly');
 var tableUtil     = require('../lib/tableUtil');
-var {addToSort}   = require('../lib/sortUtil');
+var {addToSort, sortBy}   = require('../lib/sortUtil');
 
 var commands      = require('./commands');
 var viewCommon    = require('./viewCommon');
@@ -260,16 +261,16 @@ GridView.gridCommands = {
   paste: function(pasteObj, cutCallback) { return this.paste(pasteObj, cutCallback); },
   cancel: function() { this.clearSelection(); },
   sortAsc: function() {
-    this.viewSection.activeSortSpec.assign([this.currentColumn().getRowId()]);
+    sortBy(this.viewSection.activeSortSpec, this.currentColumn().getRowId(), Sort.ASC);
   },
   sortDesc: function() {
-    this.viewSection.activeSortSpec.assign([-this.currentColumn().getRowId()]);
+    sortBy(this.viewSection.activeSortSpec, this.currentColumn().getRowId(), Sort.DESC);
   },
   addSortAsc: function() {
-    addToSort(this.viewSection.activeSortSpec, this.currentColumn().getRowId());
+    addToSort(this.viewSection.activeSortSpec, this.currentColumn().getRowId(), Sort.ASC);
   },
   addSortDesc: function() {
-    addToSort(this.viewSection.activeSortSpec, -this.currentColumn().getRowId());
+    addToSort(this.viewSection.activeSortSpec, this.currentColumn().getRowId(), Sort.DESC);
   },
   toggleFreeze: function() {
     // get column selection

app/client/components/ViewConfigTab.js

@@ -9,19 +9,21 @@ var SummaryConfig = require('./SummaryConfig');
 var commands = require('./commands');
 var {CustomSectionElement} = require('../lib/CustomSectionElement');
 const {ChartConfig} = require('./ChartView');
-const {Computed, dom: grainjsDom, makeTestId, Observable, styled} = require('grainjs');
+const {Computed, dom: grainjsDom, makeTestId, Observable, styled, MultiHolder} = require('grainjs');
 
-const {addToSort, flipColDirection, parseSortColRefs} = require('app/client/lib/sortUtil');
+const {addToSort} = require('app/client/lib/sortUtil');
-const {reorderSortRefs, updatePositions} = require('app/client/lib/sortUtil');
+const {updatePositions} = require('app/client/lib/sortUtil');
 const {attachColumnFilterMenu} = require('app/client/ui/ColumnFilterMenu');
 const {addFilterMenu} = require('app/client/ui/FilterBar');
 const {cssIcon, cssRow} = require('app/client/ui/RightPanel');
 const {VisibleFieldsConfig} = require('app/client/ui/VisibleFieldsConfig');
 const {basicButton, primaryButton} = require('app/client/ui2018/buttons');
+const {labeledLeftSquareCheckbox} = require("app/client/ui2018/checkbox");
 const {colors} = require('app/client/ui2018/cssVars');
 const {cssDragger} = require('app/client/ui2018/draggableList');
 const {menu, menuItem, select} = require('app/client/ui2018/menus');
 const {confirmModal} = require('app/client/ui2018/modals');
+const {Sort} = require('app/common/SortSpec');
 const isEqual = require('lodash/isEqual');
 const {cssMenuItem} = require('popweasel');
 
@@ -207,7 +209,7 @@ ViewConfigTab.prototype.buildSortDom = function() {
 
     // Computed to indicate if sort has changed from saved.
     const hasChanged = Computed.create(null, (use) =>
-      !isEqual(use(section.activeSortSpec), parseSortColRefs(use(section.sortColRefs))));
+      !isEqual(use(section.activeSortSpec), Sort.parseSortColRefs(use(section.sortColRefs))));
 
     // Computed array of sortable columns.
     const columns = Computed.create(null, (use) => {
@@ -217,26 +219,36 @@ ViewConfigTab.prototype.buildSortDom = function() {
                  .map(col => ({
                    label: use(col.colId),
                    value: col.getRowId(),
-                   icon: 'FieldColumn'
+                   icon: 'FieldColumn',
+                   type: col.type()
                  }));
     });
 
-    // KoArray of sortRows used to create the draggableList.
+    // We only want to recreate rows, when the actual columns change.
-    const sortRows = koArray.syncedKoArray(section.activeSortSpec);
+    const colRefs = Computed.create(null, (use) => {
+      return use(section.activeSortSpec).map(col => Sort.getColRef(col));
+    });
+    const sortRows = koArray(colRefs.get());
+    colRefs.addListener((curr, prev) => {
+      if (!isEqual(curr, prev)){
+        sortRows.assign(curr);
+      }
+    })
 
     // Sort row create function for each sort row in the draggableList.
-    const rowCreateFn = sortRef =>
+    const rowCreateFn = colRef =>
-      this._buildSortRow(sortRef, section.activeSortSpec.peek(), columns);
+      this._buildSortRow(colRef, section.activeSortSpec, columns);
 
     // Reorder function called when sort rows are reordered via dragging.
     const reorder = (...args) => {
-      const spec = reorderSortRefs(section.activeSortSpec.peek(), ...args);
+      const spec = Sort.reorderSortRefs(section.activeSortSpec.peek(), ...args);
       this._saveSort(spec);
     };
 
     return grainjsDom('div',
       grainjsDom.autoDispose(hasChanged),
       grainjsDom.autoDispose(columns),
+      grainjsDom.autoDispose(colRefs),
       grainjsDom.autoDispose(sortRows),
       // Sort rows.
       kf.draggableList(sortRows, rowCreateFn, {
@@ -280,46 +292,101 @@ ViewConfigTab.prototype.buildSortDom = function() {
 };
 
 // Builds a single row of the sort dom
-// Takes the sortRef (signed colRef), current sortSpec and array of column select options to show
+// Takes the colRef, current sortSpec and array of column select options to show
 // in the column select dropdown.
-ViewConfigTab.prototype._buildSortRow = function(sortRef, sortSpec, columns) {
+ViewConfigTab.prototype._buildSortRow = function(colRef, sortSpec, columns) {
-  // sortRef is a rowId of a column or its negative value (indicating descending order).
+  const holder = new MultiHolder();
-  const colRef = Math.abs(sortRef);
+
-  // Computed to show the selected column at the sortSpec index and to update the
+  const col           = Computed.create(holder, () => colRef);
-  // sortSpec on write.
+  const details       = Computed.create(holder, (use) => Sort.specToDetails(Sort.findCol(use(sortSpec), colRef)));
-  const col = Computed.create(null, () => colRef);
+  const hasSpecs      = Computed.create(holder, details, (_, details) => Sort.hasOptions(details));
+  const isAscending   = Computed.create(holder, details, (_, details) => details.direction === Sort.ASC);
+
   col.onWrite((newRef) => {
-    const idx = sortSpec.findIndex(_sortRef => _sortRef === sortRef);
+    let specs = sortSpec.peek();
-    const swapIdx = sortSpec.findIndex(_sortRef => Math.abs(_sortRef) === newRef);
+    const colSpec = Sort.findCol(specs, colRef);
-    // If the selected ref is already present, swap it with the old ref.
+    const newSpec = Sort.findCol(specs, newRef);
-    // Maintain sort order in each case for simplicity.
+    if (newSpec) {
-    if (swapIdx > -1) { sortSpec.splice(swapIdx, 1, sortSpec[swapIdx] > 0 ? colRef : -colRef); }
+      // this column is already there so only swap order
-    if (colRef !== newRef) { sortSpec.splice(idx, 1, sortRef > 0 ? newRef : -newRef); }
+      specs = Sort.swap(specs, colRef, newRef);
-    this._saveSort(sortSpec);
+      // but keep the directions
+      specs = Sort.setSortDirection(specs, colRef, Sort.direction(newSpec))
+      specs = Sort.setSortDirection(specs, newRef, Sort.direction(colSpec))
+    } else {
+      specs = Sort.replace(specs, colRef, Sort.createColSpec(newRef, Sort.direction(colSpec)));
+    }
+    this._saveSort(specs);
   });
+
+  const computedFlag = (flag, allowedTypes, label) => {
+    const computed = Computed.create(holder, details, (_, details) => details[flag] || false);
+    computed.onWrite(value => {
+      const specs = sortSpec.peek();
+      // Get existing details
+      const details = Sort.specToDetails(Sort.findCol(specs, colRef));
+      // Update flags
+      details[flag] = value;
+      // Replace the colSpec at the index
+      this._saveSort(Sort.replace(specs, Sort.getColRef(colRef), details));
+    });
+    return {computed, allowedTypes, flag, label};
+  }
+  const orderByChoice = computedFlag('orderByChoice', ['Choice'], 'Use choice position');
+  const naturalSort   = computedFlag('naturalSort', ['Text'], 'Natural sort');
+  const emptyLast     = computedFlag('emptyLast', null, 'Empty values last');
+  const flags = [orderByChoice, emptyLast, naturalSort];
+
+  const column = columns.get().find(col => col.value === Sort.getColRef(colRef));
+
   return cssSortRow(
-    grainjsDom.autoDispose(col),
+    grainjsDom.autoDispose(holder),
     cssSortSelect(
       select(col, columns)
     ),
-    cssSortIconPrimaryBtn('Sort',
+    // Use domComputed method for this icon, for dynamic testId, otherwise
-      grainjsDom.style('transform', sortRef < 0 ? 'none' : 'scaleY(-1)'),
+    // we are not able add it dynamically.
-      grainjsDom.on('click', () => {
+    grainjsDom.domComputed(isAscending, isAscending =>
-        this._saveSort(flipColDirection(sortSpec, sortRef));
+      cssSortIconPrimaryBtn(
+        "Sort",
+        grainjsDom.style("transform", isAscending ? "scaleY(-1)" : "none"),
+        grainjsDom.on("click", () => {
+          this._saveSort(Sort.flipSort(sortSpec.peek(), colRef));
         }),
-      testId('sort-order'),
+        testId("sort-order"),
-      testId(sortRef < 0 ? 'sort-order-desc' : 'sort-order-asc')
+        testId(isAscending ? "sort-order-asc" : "sort-order-desc")
+      )
     ),
     cssSortIconBtn('Remove',
       grainjsDom.on('click', () => {
-        const _idx = sortSpec.findIndex(c => c === sortRef);
+        const specs = sortSpec.peek();
-        if (_idx !== -1) {
+        if (Sort.findCol(specs, colRef)) {
-          sortSpec.splice(_idx, 1);
+          this._saveSort(Sort.removeCol(specs, colRef));
-          this._saveSort(sortSpec);
         }
       }),
       testId('sort-remove')
     ),
+    cssMenu(
+      cssBigIconWrapper(
+        cssIcon('Dots', grainjsDom.cls(cssBgLightGreen.className, hasSpecs)),
+        testId('sort-options-icon'),
+      ),
+      menu(_ctl => flags.map(({computed, allowedTypes, flag, label}) => {
+        // when allowedTypes is null, flag can be used for every column
+        const enabled = !allowedTypes || allowedTypes.includes(column.type);
+        return cssMenuItem(
+            labeledLeftSquareCheckbox(
+              computed,
+              label,
+              grainjsDom.prop('disabled', !enabled),
+            ),
+            grainjsDom.cls(cssOptionMenuItem.className),
+            grainjsDom.cls('disabled', !enabled),
+            testId('sort-option'),
+            testId(`sort-option-${flag}`),
+          );
+        },
+      ))
+    ),
     testId('sort-row')
   );
 };
@@ -329,32 +396,36 @@ ViewConfigTab.prototype._buildSortRow = function(sortRef, sortSpec, columns) {
 ViewConfigTab.prototype._buildAddToSortBtn = function(columns) {
   // Observable indicating whether the add new column row is visible.
   const showAddNew = Observable.create(null, false);
+  const available = Computed.create(null, (use) => {
+    const currentSection = use(this.activeSectionData).section;
+    const currentSortSpec = use(currentSection.activeSortSpec);
+    const specRowIds = new Set(currentSortSpec.map(_sortRef => Sort.getColRef(_sortRef)));
+    return use(columns)
+      .filter(_col => !specRowIds.has(_col.value))
+  });
   return [
     // Add column button.
     cssRow(
       grainjsDom.autoDispose(showAddNew),
+      grainjsDom.autoDispose(available),
       cssTextBtn(
         cssPlusIcon('Plus'), 'Add Column',
         testId('sort-add')
       ),
-      grainjsDom.hide(showAddNew),
+      grainjsDom.hide((use) => use(showAddNew) || !use(available).length),
       grainjsDom.on('click', () => { showAddNew.set(true); }),
     ),
     // Fake add column row that appears only when the menu is open to select a new column
     // to add to the sort. Immediately destroyed when menu is closed.
-    grainjsDom.maybe((use) => use(showAddNew) && use(columns), _columns => {
+    grainjsDom.maybe((use) => use(showAddNew) && use(available), _columns => {
       const col = Observable.create(null, 0);
       const currentSection = this.activeSectionData().section;
-      const currentSortSpec = currentSection.activeSortSpec();
-      const specRowIds = new Set(currentSortSpec.map(_sortRef => Math.abs(_sortRef)));
       // Function called when a column select value is clicked.
       const onClick = (_col) => {
         showAddNew.set(false); // Remove add row ASAP to prevent flickering
-        addToSort(currentSection.activeSortSpec, _col.value);
+        addToSort(currentSection.activeSortSpec, _col.value, 1);
       };
-      const menuCols = _columns
+      const menuCols = _columns.map(_col =>
-        .filter(_col => !specRowIds.has(_col.value))
-        .map(_col =>
         menuItem(() => onClick(_col),
           cssMenuIcon(_col.icon),
           _col.label,
@@ -380,7 +451,8 @@ ViewConfigTab.prototype._buildAddToSortBtn = function(columns) {
         cssSortIconPrimaryBtn('Sort',
           grainjsDom.style('transform', 'scaleY(-1)')
         ),
-        cssSortIconBtn('Remove')
+        cssSortIconBtn('Remove'),
+        cssBigIconWrapper(cssIcon('Dots')),
       ));
     })
   ];
@@ -820,4 +892,39 @@ const cssNoMarginLeft = styled('div', `
 
 const cssIconWrapper = styled('div', ``);
 
+const cssBigIconWrapper = styled('div', `
+  padding: 3px;
+  border-radius: 3px;
+  cursor: pointer;
+  user-select: none;
+`);
+
+const cssMenu = styled('div', `
+  display: inline-flex;
+  cursor: pointer;
+  border-radius: 3px;
+  border: 1px solid transparent;
+  &:hover, &.weasel-popup-open {
+    background-color: ${colors.mediumGrey};
+  }
+`);
+
+const cssBgLightGreen = styled(`div`, `
+  background: ${colors.lightGreen}
+`)
+
+const cssOptionMenuItem = styled('div', `
+  &:hover {
+    background-color: ${colors.mediumGrey};
+  }
+  & label {
+    flex: 1;
+    cursor: pointer;
+  }
+  &.disabled * {
+    color: ${colors.darkGrey} important;
+    cursor: not-allowed;
+  }
+`)
+
 module.exports = ViewConfigTab;

app/client/lib/sortUtil.ts

@@ -1,66 +1,34 @@
-import {GristDoc} from 'app/client/components/GristDoc';
+import { GristDoc } from 'app/client/components/GristDoc';
-import {ClientColumnGetters} from 'app/client/models/ClientColumnGetters';
+import { ClientColumnGetters } from 'app/client/models/ClientColumnGetters';
-import {ViewSectionRec} from 'app/client/models/entities/ViewSectionRec';
+import { ViewSectionRec } from 'app/client/models/entities/ViewSectionRec';
 import * as rowset from 'app/client/models/rowset';
-import {MANUALSORT} from 'app/common/gristTypes';
+import { MANUALSORT } from 'app/common/gristTypes';
-import {SortFunc} from 'app/common/SortFunc';
+import { SortFunc } from 'app/common/SortFunc';
+import { Sort } from 'app/common/SortSpec';
 import * as ko from 'knockout';
 import range = require('lodash/range');
 
+
 /**
  * Adds a column to the given sort spec, replacing its previous occurrence if
  * it's already in the sort spec.
  */
-export function addToSort(sortSpecObs: ko.Observable<number[]>, colRef: number) {
+export function addToSort(sortSpecObs: ko.Observable<Sort.SortSpec>, colRef: number, direction: -1|1) {
   const spec = sortSpecObs.peek();
-  const index = spec.findIndex((colRefSpec) => Math.abs(colRefSpec) === Math.abs(colRef));
+  const index = Sort.findColIndex(spec, colRef);
   if (index !== -1) {
-    spec.splice(index, 1, colRef);
+    spec.splice(index, 1, colRef * direction);
   } else {
-    spec.push(colRef);
+    spec.push(colRef * direction);
   }
   sortSpecObs(spec);
 }
 
-
+export function sortBy(sortSpecObs: ko.Observable<Sort.SortSpec>, colRef: number, direction: -1|1) {
-// Takes an activeSortSpec and sortRef to flip (negative sortRefs signify descending order) and returns a new
+  let spec = sortSpecObs.peek();
-// activeSortSpec with that sortRef flipped (or original spec if sortRef not found).
+  const colSpec = Sort.findCol(spec, colRef) ?? colRef;
-export function flipColDirection(spec: number[], sortRef: number): number[] {
+  spec = [Sort.setColDirection(colSpec, direction)];
-  const idx = spec.findIndex(c => c === sortRef);
+  sortSpecObs(spec);
-  if (idx !== -1) {
-    const newSpec = Array.from(spec);
-    newSpec[idx] *= -1;
-    return newSpec;
-  }
-  return spec;
-}
-
-// Parses the sortColRefs string, defaulting to an empty array on invalid input.
-export function parseSortColRefs(sortColRefs: string): number[] {
-  try {
-    return JSON.parse(sortColRefs);
-  } catch (err) {
-    return [];
-  }
-}
-
-// Given the current sort spec, moves sortRef to be immediately before nextSortRef. Moves sortRef
-// to the end of the sort spec if nextSortRef is null.
-// If the given sortRef or nextSortRef cannot be found, return sortSpec unchanged.
-export function reorderSortRefs(spec: number[], sortRef: number, nextSortRef: number|null): number[] {
-  const updatedSpec = spec.slice();
-
-  // Remove sortRef from sortSpec.
-  const _idx = updatedSpec.findIndex(c => c === sortRef);
-  if (_idx === -1) { return spec; }
-  updatedSpec.splice(_idx, 1);
-
-  // Add sortRef to before nextSortRef
-  const _nextIdx = nextSortRef ? updatedSpec.findIndex(c => c === nextSortRef) : updatedSpec.length;
-  if (_nextIdx === -1) { return spec; }
-  updatedSpec.splice(_nextIdx, 0, sortRef);
-
-  return updatedSpec;
 }
 
 // Updates the manual sort positions to the positions currently displayed in the view, sets the
@@ -72,21 +40,27 @@ export async function updatePositions(gristDoc: GristDoc, section: ViewSectionRe
 
   // Build a sorted array of rowIds the way a view would, using the active sort spec. We just need
   // the sorted list, and can dispose the observable array immediately.
-  const sortFunc = new SortFunc(new ClientColumnGetters(tableModel, {unversioned: true}));
+  const sortFunc = new SortFunc(new ClientColumnGetters(tableModel, { unversioned: true }));
   sortFunc.updateSpec(section.activeDisplaySortSpec.peek());
-  const sortedRows = rowset.SortedRowSet.create(null, (a: rowset.RowId, b: rowset.RowId) =>
+  const sortedRows = rowset.SortedRowSet.create(
-    sortFunc.compare(a as number, b as number), tableModel.tableData);
+    null,
+    (a: rowset.RowId, b: rowset.RowId) => sortFunc.compare(a as number, b as number),
+    tableModel.tableData
+  );
   sortedRows.subscribeTo(tableModel);
   const sortedRowIds = sortedRows.getKoArray().peek().slice(0);
   sortedRows.dispose();
 
   // The action just assigns consecutive positions to the sorted rows.
   const colInfo = {[MANUALSORT]: range(0, sortedRowIds.length)};
-  await gristDoc.docData.sendActions([
+  await gristDoc.docData.sendActions(
+    [
       // Update row positions and clear the saved sort spec as a single action bundle.
       ['BulkUpdateRecord', tableId, sortedRowIds, colInfo],
-    ['UpdateRecord', '_grist_Views_section', section.getRowId(), {sortColRefs: '[]'}]
+      ['UpdateRecord', '_grist_Views_section', section.getRowId(), {sortColRefs: '[]'}],
-  ], `Updated table ${tableId} row positions.`);
+    ],
+    `Updated table ${tableId} row positions.`
+  );
   // Finally clear out the local sort spec.
   section.activeSortJson.revert();
 }

app/client/models/ClientColumnGetters.ts

@@ -1,6 +1,8 @@
 import * as DataTableModel from 'app/client/models/DataTableModel';
-import {ColumnGetters} from 'app/common/ColumnGetters';
+import { ColumnGetter, ColumnGetters } from 'app/common/ColumnGetters';
 import * as gristTypes from 'app/common/gristTypes';
+import { choiceGetter } from 'app/common/SortFunc';
+import { Sort } from 'app/common/SortSpec';
 
 /**
  *
@@ -18,13 +20,15 @@ export class ClientColumnGetters implements ColumnGetters {
     unversioned?: boolean} = {}) {
   }
 
-  public getColGetter(colRef: number): ((rowId: number) => any) | null {
+  public getColGetter(colSpec: Sort.ColSpec): ColumnGetter | null {
-    const colId = this._tableModel.docModel.columns.getRowModel(Math.abs(colRef)).colId();
+    const rowModel = this._tableModel.docModel.columns.getRowModel(Sort.getColRef(colSpec));
-    const getter = this._tableModel.tableData.getRowPropFunc(colId);
+    const colId = rowModel.colId();
-    if (!getter) { return getter || null; }
+    let getter: ColumnGetter|undefined = this._tableModel.tableData.getRowPropFunc(colId);
+    if (!getter) { return null; }
     if (this._options.unversioned && this._tableModel.tableData.mayHaveVersions()) {
-      return (rowId) => {
+      const valueGetter = getter;
-        const value = getter(rowId);
+      getter = (rowId) => {
+        const value = valueGetter(rowId);
         if (value && gristTypes.isVersions(value)) {
           const versions = value[1];
           return ('parent' in versions) ? versions.parent :
@@ -33,6 +37,13 @@ export class ClientColumnGetters implements ColumnGetters {
         return value;
       };
     }
+    const details = Sort.specToDetails(colSpec);
+    if (details.orderByChoice) {
+      if (rowModel.pureType() === 'Choice') {
+        const choices: string[] = rowModel.widgetOptionsJson.peek()?.choices || [];
+        getter = choiceGetter(getter, choices);
+      }
+    }
     return getter;
   }
 

app/client/models/entities/ViewSectionRec.ts

@@ -1,13 +1,14 @@
 import * as BaseView from 'app/client/components/BaseView';
-import {CursorPos} from 'app/client/components/Cursor';
+import { ColumnRec, TableRec, ViewFieldRec, ViewRec } from 'app/client/models/DocModel';
-import {KoArray} from 'app/client/lib/koArray';
-import {ColumnRec, TableRec, ViewFieldRec, ViewRec} from 'app/client/models/DocModel';
-import {DocModel, IRowModel, recordSet, refRecord} from 'app/client/models/DocModel';
 import * as modelUtil from 'app/client/models/modelUtil';
-import {RowId} from 'app/client/models/rowset';
-import {getWidgetTypes} from 'app/client/ui/widgetTypes';
-import {Computed} from 'grainjs';
 import * as ko from 'knockout';
+import { CursorPos, } from 'app/client/components/Cursor';
+import { KoArray, } from 'app/client/lib/koArray';
+import { DocModel, IRowModel, recordSet, refRecord, } from 'app/client/models/DocModel';
+import { RowId, } from 'app/client/models/rowset';
+import { getWidgetTypes, } from 'app/client/ui/widgetTypes';
+import { Sort, } from 'app/common/SortSpec';
+import { Computed, } from 'grainjs';
 import defaults = require('lodash/defaults');
 
 // Represents a section of user views, now also known as a "page widget" (e.g. a view may contain
@@ -44,10 +45,10 @@ export interface ViewSectionRec extends IRowModel<"_grist_Views_section"> {
 
   // is an array (parsed from JSON) of colRefs (i.e. rowIds into the columns table), with a
   // twist: a rowId may be positive or negative, for ascending or descending respectively.
-  activeSortSpec: modelUtil.ObjObservable<number[]>;
+  activeSortSpec: modelUtil.ObjObservable<Sort.SortSpec>;
 
   // Modified sort spec to take into account any active display columns.
-  activeDisplaySortSpec: ko.Computed<number[]>;
+  activeDisplaySortSpec: ko.Computed<Sort.SortSpec>;
 
   // Evaluates to an array of column models, which are not referenced by anything in viewFields.
   hiddenColumns: ko.Computed<ColumnRec[]>;
@@ -209,9 +210,9 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel):
   // twist: a rowId may be positive or negative, for ascending or descending respectively.
   // TODO: This method of ignoring columns which are deleted is inefficient and may cause conflicts
   //  with sharing.
-  this.activeSortSpec = modelUtil.jsonObservable(this.activeSortJson, (obj: any) => {
+  this.activeSortSpec = modelUtil.jsonObservable(this.activeSortJson, (obj: Sort.SortSpec|null) => {
-    return (obj || []).filter((sortRef: number) => {
+    return (obj || []).filter((sortRef: Sort.ColSpec) => {
-      const colModel = docModel.columns.getRowModel(Math.abs(sortRef));
+      const colModel = docModel.columns.getRowModel(Sort.getColRef(sortRef));
       return !colModel._isDeleted() && colModel.getRowId();
     });
   });
@@ -219,10 +220,10 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel):
   // Modified sort spec to take into account any active display columns.
   this.activeDisplaySortSpec = this.autoDispose(ko.computed(() => {
     return this.activeSortSpec().map(directionalColRef => {
-      const colRef = Math.abs(directionalColRef);
+      const colRef = Sort.getColRef(directionalColRef);
       const field = this.viewFields().all().find(f => f.column().origColRef() === colRef);
       const effectiveColRef = field ? field.displayColRef() : colRef;
-      return directionalColRef > 0 ? effectiveColRef : -effectiveColRef;
+      return Sort.swapColRef(directionalColRef, effectiveColRef);
     });
   }));
 

app/client/ui/GridViewMenus.ts

@@ -1,9 +1,10 @@
-import {allCommands} from 'app/client/components/commands';
+import { allCommands } from 'app/client/components/commands';
-import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec';
+import { ViewFieldRec } from 'app/client/models/entities/ViewFieldRec';
-import {testId, vars} from 'app/client/ui2018/cssVars';
+import { testId, vars } from 'app/client/ui2018/cssVars';
-import {icon} from 'app/client/ui2018/icons';
+import { icon } from 'app/client/ui2018/icons';
-import {menuDivider, menuItem, menuItemCmd} from 'app/client/ui2018/menus';
+import { menuDivider, menuItem, menuItemCmd } from 'app/client/ui2018/menus';
-import {dom, DomElementArg, styled} from 'grainjs';
+import { Sort } from 'app/common/SortSpec';
+import { dom, DomElementArg, styled } from 'grainjs';
 import isEqual = require('lodash/isEqual');
 
 interface IView {
@@ -47,7 +48,7 @@ interface IMultiColumnContextMenu {
 
 interface IColumnContextMenu extends IMultiColumnContextMenu {
   filterOpenFunc: () => void;
-  sortSpec: number[];
+  sortSpec: Sort.SortSpec;
   colId: number;
 }
 
@@ -75,19 +76,18 @@ export function ColumnContextMenu(options: IColumnContextMenu) {
         icon('Sort', dom.style('transform', 'scaley(-1)')),
         'A-Z',
         dom.style('flex', ''),
-        cssCustomMenuItem.cls('-selected', isEqual(sortSpec, [colId])),
+        cssCustomMenuItem.cls('-selected', Sort.containsOnly(sortSpec, colId, Sort.ASC)),
         testId('sort-asc'),
       ),
       customMenuItem(
         allCommands.sortDesc.run,
         icon('Sort'),
         'Z-A',
-        cssCustomMenuItem.cls('-selected', isEqual(sortSpec, [-colId])),
+        cssCustomMenuItem.cls('-selected', Sort.containsOnly(sortSpec, colId, Sort.DESC)),
         testId('sort-dsc'),
       ),
       testId('sort'),
     ),
-    menuDivider({style: 'margin-bottom: 0; margin-top: 0;'}),
     addToSortLabel ? [
       cssRowMenuItem(
         customMenuItem(
@@ -95,20 +95,22 @@ export function ColumnContextMenu(options: IColumnContextMenu) {
           cssRowMenuLabel(addToSortLabel, testId('add-to-sort-label')),
           icon('Sort', dom.style('transform', 'scaley(-1)')),
           'A-Z',
-          cssCustomMenuItem.cls('-selected', sortSpec.includes(colId)),
+          cssCustomMenuItem.cls('-selected', Sort.contains(sortSpec, colId, Sort.ASC)),
           testId('add-to-sort-asc'),
         ),
         customMenuItem(
           allCommands.addSortDesc.run,
           icon('Sort'),
           'Z-A',
-          cssCustomMenuItem.cls('-selected', sortSpec.includes(-colId)),
+          cssCustomMenuItem.cls('-selected', Sort.contains(sortSpec, colId, Sort.DESC)),
           testId('add-to-sort-dsc'),
         ),
         testId('add-to-sort'),
       ),
-      menuDivider({style: 'margin-top: 0;'}),
     ] : null,
+    menuDivider({style: 'margin-bottom: 0; margin-top: 0;'}),
+    menuItem(allCommands.sortFilterTabOpen.run, 'More sort options ...', testId('more-sort-options')),
+    menuDivider({style: 'margin-top: 0;'}),
     menuItemCmd(allCommands.renameField, 'Rename column', disableForReadonlyColumn),
     menuItemCmd(allCommands.hideField, 'Hide column', disableForReadonlyView),
     freezeMenuItemCmd(options),
@@ -269,8 +271,8 @@ function freezeMenuItemCmd(options: IMultiColumnContextMenu) {
 // Returns 'Add to sort' is there are columns in the sort spec but colId is not part of it. Returns
 // undefined if colId is the only column in the spec. Otherwise returns `Sorted (#N)` where #N is
 // the position (1 based) of colId in the spec.
-function getAddToSortLabel(sortSpec: number[], colId: number): string|undefined {
+function getAddToSortLabel(sortSpec: Sort.SortSpec, colId: number): string|undefined {
-  const columnsInSpec = sortSpec.map((n) => Math.abs(n));
+  const columnsInSpec = sortSpec.map((n) =>Sort.getColRef(n));
   if (sortSpec.length !== 0 && !isEqual(columnsInSpec, [colId])) {
     const index = columnsInSpec.indexOf(colId);
     if (index > -1) {

app/client/ui/ViewSectionMenu.ts

@@ -1,18 +1,18 @@
-import {flipColDirection, parseSortColRefs} from 'app/client/lib/sortUtil';
+import { reportError } from 'app/client/models/AppModel';
-import {reportError} from 'app/client/models/AppModel';
+import { ColumnRec, DocModel, ViewFieldRec, ViewRec, ViewSectionRec } from 'app/client/models/DocModel';
-import {ColumnRec, DocModel, ViewFieldRec, ViewRec, ViewSectionRec} from 'app/client/models/DocModel';
+import { CustomComputed } from 'app/client/models/modelUtil';
-import {CustomComputed} from 'app/client/models/modelUtil';
+import { attachColumnFilterMenu } from 'app/client/ui/ColumnFilterMenu';
-import {attachColumnFilterMenu} from 'app/client/ui/ColumnFilterMenu';
+import { addFilterMenu } from 'app/client/ui/FilterBar';
-import {addFilterMenu} from 'app/client/ui/FilterBar';
+import { hoverTooltip } from 'app/client/ui/tooltips';
-import {hoverTooltip} from 'app/client/ui/tooltips';
+import { makeViewLayoutMenu } from 'app/client/ui/ViewLayoutMenu';
-import {makeViewLayoutMenu} from 'app/client/ui/ViewLayoutMenu';
+import { basicButton, primaryButton } from 'app/client/ui2018/buttons';
-import {basicButton, primaryButton} from 'app/client/ui2018/buttons';
+import { colors, vars } from 'app/client/ui2018/cssVars';
-import {colors, vars} from 'app/client/ui2018/cssVars';
+import { icon } from 'app/client/ui2018/icons';
-import {icon} from 'app/client/ui2018/icons';
+import { menu } from 'app/client/ui2018/menus';
-import {menu} from 'app/client/ui2018/menus';
+import { Sort } from 'app/common/SortSpec';
-import {Computed, dom, fromKo, IDisposableOwner, makeTestId, Observable, styled} from 'grainjs';
+import { Computed, dom, fromKo, IDisposableOwner, makeTestId, Observable, styled } from 'grainjs';
+import { PopupControl } from 'popweasel';
 import difference = require('lodash/difference');
-import {PopupControl} from 'popweasel';
 
 const testId = makeTestId('test-section-menu-');
 
@@ -105,29 +105,27 @@ export function viewSectionMenu(owner: IDisposableOwner, docModel: DocModel, vie
   ];
 }
 
-function makeSortPanel(section: ViewSectionRec, sortSpec: number[], getColumn: (row: number) => ColumnRec) {
+function makeSortPanel(section: ViewSectionRec, sortSpec: Sort.SortSpec, getColumn: (row: number) => ColumnRec) {
-  const changedColumns = difference(sortSpec, parseSortColRefs(section.sortColRefs.peek()));
+  const changedColumns = difference(sortSpec, Sort.parseSortColRefs(section.sortColRefs.peek()));
-  const sortColumns = sortSpec.map(colRef => {
+  const sortColumns = sortSpec.map(colSpec => {
     // colRef is a rowId of a column or its negative value (indicating descending order).
-    const col = getColumn(Math.abs(colRef));
+    const col = getColumn(Sort.getColRef(colSpec));
     return cssMenuText(
       cssMenuIconWrapper(
-        cssMenuIconWrapper.cls('-changed', changedColumns.includes(colRef)),
+        cssMenuIconWrapper.cls('-changed', changedColumns.includes(colSpec)),
-        cssMenuIconWrapper.cls(colRef < 0 ? '-desc' : '-asc'),
+        cssMenuIconWrapper.cls(Sort.isAscending(colSpec) ? '-asc' : '-desc'),
         cssIcon('Sort',
-          dom.style('transform', colRef < 0 ? 'none' : 'scaleY(-1)'),
+          dom.style('transform', Sort.isAscending(colSpec) ? 'scaleY(-1)' : 'none'),
           dom.on('click', () => {
-            section.activeSortSpec(flipColDirection(sortSpec, colRef));
+            section.activeSortSpec(Sort.flipSort(sortSpec, colSpec));
           })
         )
       ),
       cssMenuTextLabel(col.colId()),
       cssMenuIconWrapper(
         cssIcon('Remove', testId('btn-remove-sort'), dom.on('click', () => {
-          const idx = sortSpec.findIndex(c => c === colRef);
+          if (Sort.findCol(sortSpec, colSpec)) {
-          if (idx !== -1) {
+            section.activeSortSpec(Sort.removeCol(sortSpec, colSpec));
-            sortSpec.splice(idx, 1);
-            section.activeSortSpec(sortSpec);
           }
         }))
       ),

app/client/ui2018/checkbox.ts

@@ -120,33 +120,40 @@ export const cssLabelText = styled('span', `
 type CheckboxArg = DomArg<HTMLInputElement>;
 
 function checkbox(
-  obs: Observable<boolean>, cssCheckbox: typeof cssCheckboxSquare, label: DomArg = '',  ...domArgs: CheckboxArg[]
+  obs: Observable<boolean>, cssCheckbox: typeof cssCheckboxSquare,
+  label: DomArg, right: boolean,  ...domArgs: CheckboxArg[]
 ) {
-  return cssLabel(
+  const field = cssCheckbox(
-    cssCheckbox(
       { type: 'checkbox' },
       dom.prop('checked', obs),
       dom.on('change', (ev, el) => obs.set(el.checked)),
       ...domArgs
-    ),
-    label ? cssLabelText(label) : null
     );
+  const text = label ? cssLabelText(label) : null;
+  if (right) {
+    return cssReversedLabel([text, cssInlineRelative(field)]);
+  }
+  return cssLabel(field, text);
 }
 
 export function squareCheckbox(obs: Observable<boolean>, ...domArgs: CheckboxArg[]) {
-  return checkbox(obs, cssCheckboxSquare, '', ...domArgs);
+  return checkbox(obs, cssCheckboxSquare, '', false, ...domArgs);
 }
 
 export function circleCheckbox(obs: Observable<boolean>, ...domArgs: CheckboxArg[]) {
-  return checkbox(obs, cssCheckboxCircle, '', ...domArgs);
+  return checkbox(obs, cssCheckboxCircle, '', false, ...domArgs);
 }
 
 export function labeledSquareCheckbox(obs: Observable<boolean>, label: DomArg, ...domArgs: CheckboxArg[]) {
-  return checkbox(obs, cssCheckboxSquare, label, ...domArgs);
+  return checkbox(obs, cssCheckboxSquare, label, false, ...domArgs);
+}
+
+export function labeledLeftSquareCheckbox(obs: Observable<boolean>, label: DomArg, ...domArgs: CheckboxArg[]) {
+  return checkbox(obs, cssCheckboxSquare, label, true, ...domArgs);
 }
 
 export function labeledCircleCheckbox(obs: Observable<boolean>, label: DomArg, ...domArgs: CheckboxArg[]) {
-  return checkbox(obs, cssCheckboxCircle, label, ...domArgs);
+  return checkbox(obs, cssCheckboxCircle, label, false, ...domArgs);
 }
 
 export const Indeterminate = 'indeterminate';
@@ -158,7 +165,7 @@ function triStateCheckbox(
   const checkboxObs = Computed.create(null, obs, (_use, state) => state === true)
     .onWrite((checked) => obs.set(checked));
   return checkbox(
-    checkboxObs, cssCheckbox, label,
+    checkboxObs, cssCheckbox, label, false,
     dom.prop('indeterminate', (use) => use(obs) === 'indeterminate'),
     dom.autoDispose(checkboxObs),
     ...domArgs
@@ -172,3 +179,17 @@ export function triStateSquareCheckbox(obs: Observable<TriState>, ...domArgs: Ch
 export function labeledTriStateSquareCheckbox(obs: Observable<TriState>, label: string, ...domArgs: CheckboxArg[]) {
   return triStateCheckbox(obs, cssCheckboxSquare, label, ...domArgs);
 }
+
+const cssInlineRelative = styled('div', `
+  display: inline-block;
+  position: relative;
+  height: 16px;
+`);
+
+const cssReversedLabel = styled(cssLabel, `
+  justify-content: space-between;
+  gap: 8px;
+  & .${cssLabelText.className} {
+    margin: 0px;
+  }
+`);

app/common/ColumnGetters.ts

@@ -1,3 +1,5 @@
+import { Sort } from 'app/common/SortSpec';
+
 /**
  *
  * An interface for accessing the columns of a table by their
@@ -7,7 +9,6 @@
  *
  */
 export interface ColumnGetters {
-
   /**
    *
    * Takes a _grist_Tables_column ID and returns a function that maps
@@ -15,12 +16,14 @@ export interface ColumnGetters {
    * values if available, drawn from a corresponding display column.
    *
    */
-  getColGetter(colRef: number): ((rowId: number) => any) | null;
+  getColGetter(spec: Sort.ColSpec): ColumnGetter | null;
 
   /**
    *
    * Returns a getter for the manual sort column if it is available.
    *
    */
-  getManualSortGetter(): ((rowId: number) => any) | null;
+  getManualSortGetter(): ColumnGetter | null;
 }
+
+export type ColumnGetter = (rowId: number) => any;

app/common/SortFunc.ts

@@ -6,8 +6,48 @@
  * class should support freezing of row positions until the user chooses to re-sort. This is not
  * currently implemented.
  */
-import {ColumnGetters} from 'app/common/ColumnGetters';
+import {ColumnGetter, ColumnGetters} from 'app/common/ColumnGetters';
 import {localeCompare, nativeCompare} from 'app/common/gutil';
+import {Sort} from 'app/common/SortSpec';
+
+// Function that will amend column getter to return entry index instead
+// of entry value. Result will be a string padded with zeros, so the ordering
+// between types is preserved.
+export function choiceGetter(getter: ColumnGetter, choices: string[]): ColumnGetter {
+  return rowId => {
+    const value = getter(rowId);
+    const index = choices.indexOf(value);
+    return index >= 0 ? String(index).padStart(5, "0") : value;
+  };
+}
+
+type Comparator = (val1: any, val2: any) => number;
+
+/**
+ * Natural comparator based on built in method.
+ * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/localeCompare
+ */
+const collator = new Intl.Collator(undefined, {numeric: true});
+function naturalCompare(val1: any, val2: any) {
+  if (typeof val1 === 'string' && typeof val2 === 'string') {
+    return collator.compare(val1, val2);
+  }
+  return typedCompare(val1, val2);
+}
+
+/**
+ * Empty comparator will treat empty values as last.
+ */
+const emptyCompare = (next: Comparator) => (val1: any, val2: any) => {
+  if (!val1 && typeof val1 !== 'number') {
+      return 1;
+  }
+  if (!val2 && typeof val2 !== 'number') {
+    return -1;
+  }
+  return next(val1, val2);
+};
+
 
 /**
  * Compare two cell values, paying attention to types and values. Note that native JS comparison
@@ -55,31 +95,46 @@ function _arrayCompare(val1: any[], val2: any[]): number {
   return val1.length === val2.length ? 0 : -1;
 }
 
-type ColumnGetter = (rowId: number) => any;
-
 /**
  * getters is an implementation of app.common.ColumnGetters
  */
 export class SortFunc {
   // updateSpec() or updateGetters() can populate these fields, used by the compare() method.
   private _colGetters: ColumnGetter[] = [];  // Array of column getters (mapping rowId to column value)
-  private _ascFlags: number[] = [];           // Array of 1 (ascending) or -1 (descending) flags.
+  private _directions: number[] = [];           // Array of 1 (ascending) or -1 (descending) flags.
+  private _comparators: Comparator[] = [];
 
   constructor(private _getters: ColumnGetters) {}
 
-  public updateSpec(sortSpec: number[]): void {
+  public updateSpec(sortSpec: Sort.SortSpec): void {
     // Prepare an array of column getters for each column in sortSpec.
-    this._colGetters = sortSpec.map(colRef => {
+    this._colGetters = sortSpec.map(colSpec => {
-      return this._getters.getColGetter(Math.abs(colRef));
+      return this._getters.getColGetter(colSpec);
     }).filter(getter => getter) as ColumnGetter[];
 
     // Collect "ascending" flags as an array of 1 or -1, one for each column.
-    this._ascFlags = sortSpec.map(colRef => (colRef >= 0 ? 1 : -1));
+    this._directions = sortSpec.map(colSpec => Sort.direction(colSpec));
+
+    // Collect comparator functions
+    this._comparators = sortSpec.map(colSpec => {
+      const details = Sort.specToDetails(colSpec);
+      let comparator = typedCompare;
+      if (details.naturalSort) {
+        comparator = naturalCompare;
+      }
+      // Empty decorator should be added last, as first we want to compare
+      // empty values
+      if (details.emptyLast) {
+        comparator = emptyCompare(comparator);
+      }
+      return comparator;
+    });
 
     const manualSortGetter = this._getters.getManualSortGetter();
     if (manualSortGetter) {
       this._colGetters.push(manualSortGetter);
-      this._ascFlags.push(1);
+      this._directions.push(1);
+      this._comparators.push(typedCompare);
     }
   }
 
@@ -89,9 +144,12 @@ export class SortFunc {
   public compare(rowId1: number, rowId2: number): number {
     for (let i = 0, len = this._colGetters.length; i < len; i++) {
       const getter = this._colGetters[i];
-      const value = typedCompare(getter(rowId1), getter(rowId2));
+      const val1 = getter(rowId1);
-      if (value) {
+      const val2 = getter(rowId2);
-        return value * this._ascFlags[i];
+      const comparator = this._comparators[i];
+      const result = comparator(val1, val2);
+      if (result !== 0 /* not equal */) {
+        return result * this._directions[i];
       }
     }
     return nativeCompare(rowId1, rowId2);

app/common/SortSpec.ts

@@ -0,0 +1,324 @@
+/**
+ * Sort namespace provides helper function to work with sort expression.
+ *
+ * Sort expression is a list of column sort expressions, each describing how to
+ * sort particular column. Column expression can be either:
+ *
+ * - Positive number: column with matching id will be sorted in ascending order
+ * - Negative number: column will be sorted in descending order
+ * - String containing a positive number: same as above
+ * - String containing a negative number: same as above
+ * - String containing a number and sorting options:
+ *   '1:flag1;flag2;flag3'
+ *   '-1:flag1;flag2;flag3'
+ *   Sorting options modifies the sorting algorithm, supported options are:
+ *   - orderByChoice: For choice column sorting function will use choice item order
+ *                    instead of choice label text.
+ *   - emptyLast:     Treat empty values as greater than non empty (default is empty values first).
+ *   - naturalSort:   For text based columns, sorting function will compare strings with numbers
+ *                    taking their numeric value rather then text representation ('a2' before 'a11)
+ */
+export namespace Sort {
+  /**
+   * Object base representation for column expression.
+   */
+  export interface ColSpecDetails {
+    colRef: number;
+    direction: Direction;
+    orderByChoice?: boolean;
+    emptyLast?: boolean;
+    naturalSort?: boolean;
+  }
+  /**
+   * Column expression type.
+   */
+  export type ColSpec = number | string;
+  /**
+   * Sort expression type, for example [1,-2, '3:emptyLast', '-4:orderByChoice']
+   */
+  export type SortSpec = Array<ColSpec>;
+  export type Direction = 1 | -1;
+  export const ASC: Direction = 1;
+  export const DESC: Direction = -1;
+
+  const NOT_FOUND = -1;
+
+  // Flag separator
+  const FLAG_SEPARATOR = ";";
+  // Separator between colRef and sorting options.
+  const OPTION_SEPARATOR = ":";
+
+  /**
+   * Checks if column expression has any sorting options.
+   */
+  export function hasOptions(colSpec: ColSpec | ColSpecDetails): boolean {
+    if (typeof colSpec === "number") {
+      return false;
+    }
+    const details = typeof colSpec !== "object" ? specToDetails(colSpec) : colSpec;
+    return Boolean(details.emptyLast || details.naturalSort || details.orderByChoice);
+  }
+
+  /**
+   * Converts column sort expression from object representation to encoded form.
+   */
+  export function detailsToSpec(d: ColSpecDetails): ColSpec {
+    const head = `${d.direction === ASC ? "" : "-"}${d.colRef}`;
+    const tail = [];
+    if (d.emptyLast) {
+      tail.push("emptyLast");
+    }
+    if (d.naturalSort) {
+      tail.push("naturalSort");
+    }
+    if (d.orderByChoice) {
+      tail.push("orderByChoice");
+    }
+    if (!tail.length) {
+      return +head;
+    }
+    return head + (tail.length ? OPTION_SEPARATOR : "") + tail.join(FLAG_SEPARATOR);
+  }
+
+  /**
+   * Converts column expression to object representation.
+   */
+  export function specToDetails(colSpec: ColSpec): ColSpecDetails {
+    return typeof colSpec === "number"
+      ? {
+          colRef: Math.abs(colSpec),
+          direction: colSpec >= 0 ? ASC: DESC,
+        }
+      : parseColSpec(colSpec);
+  }
+
+  function parseColSpec(colString: string): ColSpecDetails {
+    const REGEX = /^(-)?(\d+)(:([\w\d;]+))?$/;
+    const match = colString.match(REGEX);
+    if (!match) {
+      throw new Error("Error parsing sort expression " + colString);
+    }
+    const [, sign, colRef, , flag] = match;
+    const flags = flag?.split(";");
+    return {
+      colRef: +colRef,
+      direction: sign === "-" ? DESC : ASC,
+      orderByChoice: flags?.includes("orderByChoice"),
+      emptyLast: flags?.includes("emptyLast"),
+      naturalSort: flags?.includes("naturalSort"),
+    };
+  }
+
+  /**
+   * Extracts colRef (column row id) from column sorting expression.
+   */
+  export function getColRef(colSpec: ColSpec) {
+    if (typeof colSpec === "number") {
+      return Math.abs(colSpec);
+    }
+    return parseColSpec(colSpec).colRef;
+  }
+
+  /**
+   * Swaps column expressions.
+   */
+  export function swap(spec: SortSpec, colA: ColSpec, colB: ColSpec): SortSpec {
+    const aIndex = findColIndex(spec, colA);
+    const bIndex = findColIndex(spec, colB);
+    if (aIndex === NOT_FOUND || bIndex === NOT_FOUND) {
+      throw new Error(`Column expressions can be found (${colA} or ${colB})`);
+    }
+    const clone = spec.slice();
+    clone[aIndex] = spec[bIndex];
+    clone[bIndex] = spec[aIndex];
+    return clone;
+  }
+
+  /**
+   * Converts column expression order.
+   */
+  export function setColDirection(colSpec: ColSpec, dir: Direction): ColSpec {
+    if (typeof colSpec === "number") {
+      return Math.abs(colSpec) * dir;
+    }
+    return detailsToSpec({ ...parseColSpec(colSpec), direction: dir });
+  }
+
+  /**
+   * Creates simple column expression.
+   */
+  export function createColSpec(colRef: number, dir: Direction): ColSpec {
+    return colRef * dir;
+  }
+
+  /**
+   * Checks if a column expression is already included in sorting spec. Doesn't check sorting options.
+   */
+  export function contains(spec: SortSpec, colSpec: ColSpec, dir: Direction) {
+    const existing = findCol(spec, colSpec);
+    return !!existing && getColRef(existing) === getColRef(colSpec) && direction(existing) === dir;
+  }
+
+  export function containsOnly(spec: SortSpec, colSpec: ColSpec, dir: Direction) {
+    return spec.length === 1 && contains(spec, colSpec, dir);
+  }
+
+  /**
+   * Checks if a column is sorted in ascending order.
+   */
+  export function isAscending(colSpec: ColSpec): boolean {
+    if (typeof colSpec === "number") {
+      return colSpec >= 0;
+    }
+    return parseColSpec(colSpec).direction === ASC;
+  }
+
+  export function direction(colSpec: ColSpec): Direction {
+    return isAscending(colSpec) ? ASC : DESC;
+  }
+
+  /**
+   * Checks if two column expressions refers to the same column.
+   */
+  export function sameColumn(colSpec: ColSpec, colRef: ColSpec): boolean {
+    return getColRef(colSpec) === getColRef(colRef);
+  }
+
+  /**
+   * Swaps column id in column expression. Primary use for display columns.
+   */
+  export function swapColRef(colSpec: ColSpec, colRef: number): ColSpec {
+    if (typeof colSpec === "number") {
+      return colSpec >= 0 ? colRef : -colRef;
+    }
+    const spec = parseColSpec(colSpec);
+    return detailsToSpec({...spec, colRef});
+  }
+
+  /**
+   * Finds an index of column expression in a sorting expression.
+   */
+  export function findColIndex(sortSpec: SortSpec, colRef: ColSpec): number {
+    return sortSpec.findIndex(colSpec => sameColumn(colSpec, colRef));
+  }
+
+  export function removeCol(sortSpec: SortSpec, colRef: ColSpec): SortSpec {
+    return sortSpec.filter(col => getColRef(col) !== getColRef(colRef));
+  }
+
+  /**
+   * Finds a column expression in sorting expression (regardless sorting option).
+   */
+  export function findCol(sortSpec: SortSpec, colRef: ColSpec): ColSpec | undefined {
+    const result = sortSpec.find(colSpec => sameColumn(colSpec, colRef));
+    return result;
+  }
+
+  /**
+   * Inserts new column sort options at the index of an existing column options (and removes the old one).
+   * If the old column can't be found it does nothing.
+   * @param colRef Column id to remove
+   * @param newSpec New column sort options to put in place of the old one.
+   */
+  export function replace(sortSpec: SortSpec, colRef: number, newSpec: ColSpec | ColSpecDetails): SortSpec {
+    const index = findColIndex(sortSpec, colRef);
+    if (index >= 0) {
+      const updated = sortSpec.slice();
+      updated[index] = typeof newSpec === "object" ? detailsToSpec(newSpec) : newSpec;
+      return updated;
+    }
+    return sortSpec;
+  }
+
+  /**
+   * Flips direction for a single column, returns a new object.
+   */
+  export function flipCol(colSpec: ColSpec): ColSpec {
+    if (typeof colSpec === "number") {
+      return -colSpec;
+    }
+    const spec = parseColSpec(colSpec);
+    return detailsToSpec({ ...spec, direction: spec.direction === ASC ? DESC : ASC });
+  }
+
+  // Takes an activeSortSpec and sortRef to flip and returns a new
+  // activeSortSpec with that sortRef flipped (or original spec if sortRef not found).
+  export function flipSort(spec: SortSpec, colSpec: ColSpec): SortSpec {
+    const idx = findColIndex(spec, getColRef(colSpec));
+    if (idx !== NOT_FOUND) {
+      const newSpec = Array.from(spec);
+      newSpec[idx] = flipCol(newSpec[idx]);
+      return newSpec;
+    }
+    return spec;
+  }
+
+  export function setSortDirection(spec: SortSpec, colSpec: ColSpec, dir: Direction): SortSpec {
+    const idx = findColIndex(spec, getColRef(colSpec));
+    if (idx !== NOT_FOUND) {
+      const newSpec = Array.from(spec);
+      newSpec[idx] = setColDirection(newSpec[idx], dir);
+      return newSpec;
+    }
+    return spec;
+  }
+
+  // Parses the sortColRefs string, defaulting to an empty array on invalid input.
+  export function parseSortColRefs(sortColRefs: string): SortSpec {
+    try {
+      return JSON.parse(sortColRefs);
+    } catch (err) {
+      return [];
+    }
+  }
+
+  // Given the current sort spec, moves colSpec to be immediately before nextColSpec. Moves v
+  // to the end of the sort spec if nextColSpec is null.
+  // If the given colSpec or nextColSpec cannot be found, return sortSpec unchanged.
+  // ColSpec are identified only by colRef (order or options don't matter).
+  export function reorderSortRefs(spec: SortSpec, colSpec: ColSpec, nextColSpec: ColSpec | null): SortSpec {
+    const updatedSpec = spec.slice();
+
+    // Remove sortRef from sortSpec.
+    const _idx = findColIndex(updatedSpec, colSpec);
+    if (_idx === NOT_FOUND) {
+      return spec;
+    }
+    updatedSpec.splice(_idx, 1);
+
+    // Add sortRef to before nextSortRef
+    const _nextIdx = nextColSpec ? findColIndex(updatedSpec, nextColSpec) : updatedSpec.length;
+    if (_nextIdx === NOT_FOUND) {
+      return spec;
+    }
+    updatedSpec.splice(_nextIdx, 0, colSpec);
+
+    return updatedSpec;
+  }
+
+  // Helper function for query based sorting, which uses column names instead of columns ids.
+  // Translates expressions like -Pet, to an colRef expression like -1.
+  // NOTE: For column with zero index, it will return a string.
+  export function parseNames(sort: string[], colIdToRef: Map<string, number>): SortSpec {
+    const COL_SPEC_REG = /^(-)?([\w]+)(:.+)?/;
+    return sort.map((colSpec) => {
+      const match = colSpec.match(COL_SPEC_REG);
+      if (!match) {
+        throw new Error(`unknown key ${colSpec}`);
+      }
+      const [, sign, key, options] = match;
+      let colRef = Number(key);
+      if (!isNaN(colRef)) {
+        // This might be valid colRef
+        if (![...colIdToRef.values()].includes(colRef)) {
+          throw new Error(`invalid column id ${key}`);
+        }
+      } else if (!colIdToRef.has(key)) {
+        throw new Error(`unknown key ${key}`);
+      } else {
+        colRef = colIdToRef.get(key)!;
+      }
+      return `${sign || ""}${colRef}${options ?? ""}`;
+    });
+  }
+}

app/server/lib/ActiveDoc.ts

@@ -842,7 +842,10 @@ export class ActiveDoc extends EventEmitter {
    * @returns {Promise<TableRecordValue[]>} Records containing metadata about the visible columns
    * from `tableId`.
    */
-  public async getTableCols(docSession: OptDocSession, tableId: string): Promise<TableRecordValue[]> {
+  public async getTableCols(
+    docSession: OptDocSession,
+    tableId: string,
+    includeHidden = false): Promise<TableRecordValue[]> {
     const metaTables = await this.fetchMetaTables(docSession);
     const tableRef = tableIdToRef(metaTables, tableId);
     const [, , colRefs, columnData] = metaTables._grist_Tables_column;
@@ -852,12 +855,11 @@ export class ActiveDoc extends EventEmitter {
 
     const columns: TableRecordValue[] = [];
     (columnData.colId as string[]).forEach((id, index) => {
-      if (
+      const hasNoId = !id;
-        // TODO param to include hidden columns
+      const isHidden = hasNoId || id === "manualSort" || id.startsWith("gristHelper_");
-        id === "manualSort" || id.startsWith("gristHelper_") || !id ||
+      const fromDifferentTable = columnData.parentId[index] !== tableRef;
-        // Filter columns from the requested table
+      const skip = (isHidden && !includeHidden) || hasNoId || fromDifferentTable;
-        columnData.parentId[index] !== tableRef
+      if (skip) {
-      ) {
         return;
       }
       const column: TableRecordValue = { id, fields: { colRef: colRefs[index] } };

app/server/lib/DocApi.ts

@@ -2,7 +2,7 @@ import { createEmptyActionSummary } from "app/common/ActionSummary";
 import { ApiError } from 'app/common/ApiError';
 import { BrowserSettings } from "app/common/BrowserSettings";
 import {
-  BulkColValues, CellValue, fromTableDataAction, TableColValues, TableRecordValue,
+  BulkColValues, CellValue, ColValues, fromTableDataAction, TableColValues, TableRecordValue,
 } from 'app/common/DocActions';
 import {isRaisedException} from "app/common/gristTypes";
 import { arrayRepeat, isAffirmative } from "app/common/gutil";
@@ -45,6 +45,8 @@ import * as path from 'path';
 import * as uuidv4 from "uuid/v4";
 import * as t from "ts-interface-checker";
 import { Checker } from "ts-interface-checker";
+import { ServerColumnGetters } from 'app/server/lib/ServerColumnGetters';
+import { Sort } from 'app/common/SortSpec';
 
 // Cap on the number of requests that can be outstanding on a single document via the
 // rest doc api.  When this limit is exceeded, incoming requests receive an immediate
@@ -153,12 +155,17 @@ export class DocWorkerApi {
         throw new ApiError("Invalid query: filter values must be arrays", 400);
       }
       const tableId = req.params.tableId;
+      const session = docSessionFromRequest(req);
       const tableData = await handleSandboxError(tableId, [], activeDoc.fetchQuery(
-        docSessionFromRequest(req), {tableId, filters}, !immediate));
+        session, {tableId, filters}, !immediate));
+      // For metaTables we don't need to specify columns, search will infer it from the sort expression.
+      const isMetaTable = tableId.startsWith('_grist');
+      const columns = isMetaTable ? null :
+        await handleSandboxError('', [], activeDoc.getTableCols(session, tableId, true));
+      const params = getQueryParameters(req);
       // Apply sort/limit parameters, if set.  TODO: move sorting/limiting into data engine
       // and sql.
-      const params = getQueryParameters(req);
+      return applyQueryParameters(fromTableDataAction(tableData), params, columns);
-      return applyQueryParameters(fromTableDataAction(tableData), params);
     }
 
     // Get the specified table in column-oriented format
@@ -945,8 +952,9 @@ async function handleSandboxError<T>(tableId: string, colNames: string[], p: Pro
  * results returned to the user.
  */
 export interface QueryParameters {
-  sort?: string[];  // Columns to sort by (ascending order by default,
+  sort?: string[];  // Columns names to sort by (ascending order by default,
-                    // prepend "-" for descending order).
+                    // prepend "-" for descending order, can contain flags,
+                    // see more in Sort.SortSpec).
   limit?: number;   // Limit on number of rows to return.
 }
 
@@ -992,29 +1000,51 @@ function getQueryParameters(req: Request): QueryParameters {
 /**
  * Sort table contents being returned.  Sort keys with a '-' prefix
  * are sorted in descending order, otherwise ascending.  Contents are
- * modified in place.
+ * modified in place. Sort keys can contain sort options.
+ * Columns can be either expressed as a colId (name string) or as colRef (rowId number).
  */
-function applySort(values: TableColValues, sort: string[]) {
+function applySort(
+  values: TableColValues,
+  sort: string[],
+  _columns: TableRecordValue[]|null = null) {
   if (!sort) { return values; }
-  const sortKeys = sort.map(key => key.replace(/^-/, ''));
+
-  const iteratees = sortKeys.map(key => {
+  // First we need to prepare column description in ColValue format (plain objects).
-    if (!(key in values)) {
+  // This format is used by ServerColumnGetters.
-      throw new Error(`unknown key ${key}`);
+  let properColumns: ColValues[] = [];
+
+  // We will receive columns information only for user tables, not for metatables. So
+  // if this is the case, we will infer them from the result.
+  if (!_columns) {
+    _columns = Object.keys(values).map((col, index) => ({ id: col, fields: { colRef: index }}));
   }
-    const col = values[key];
+  // For user tables, we will not get id column (as this column is not in the schema), so we need to
-    return (i: number) => col[i];
+  // make sure the column is there.
-  });
+  else {
-  const sortSpec = sort.map((key, i) => (key.startsWith('-') ? -i - 1 : i + 1));
+    // This is enough information for ServerGetters
-  const index = values.id.map((_, i) => i);
+    _columns = [..._columns, { id : 'id', fields: {colRef: 0 }}];
-  const sortFunc = new SortFunc({
+  }
-    getColGetter(i) { return iteratees[i - 1]; },
+
-    getManualSortGetter() { return null; }
+  // Once we have proper columns, we can convert them to format that ServerColumnGetters
-  });
+  // understand.
-  sortFunc.updateSpec(sortSpec);
+  properColumns = _columns.map(c => ({
-  index.sort(sortFunc.compare.bind(sortFunc));
+    ...c.fields,
+    id : c.fields.colRef,
+    colId: c.id
+  }));
+
+  // We will sort row indices in the values object, not rows ids.
+  const rowIndices = values.id.map((__, i) => i);
+  const getters = new ServerColumnGetters(rowIndices, values, properColumns);
+  const sortFunc = new SortFunc(getters);
+  const colIdToRef = new Map(properColumns.map(({id, colId}) => [colId as string, id as number]));
+  sortFunc.updateSpec(Sort.parseNames(sort, colIdToRef));
+  rowIndices.sort(sortFunc.compare.bind(sortFunc));
+
+  // Sort resulting values according to the sorted index.
   for (const key of Object.keys(values)) {
     const col = values[key];
-    values[key] = index.map(i => col[i]);
+    values[key] = rowIndices.map(i => col[i]);
   }
   return values;
 }
@@ -1034,8 +1064,11 @@ function applyLimit(values: TableColValues, limit: number) {
 /**
  * Apply query parameters to table contents.  Contents are modified in place.
  */
-export function applyQueryParameters(values: TableColValues, params: QueryParameters): TableColValues {
+export function applyQueryParameters(
-  if (params.sort) { applySort(values, params.sort); }
+  values: TableColValues,
+  params: QueryParameters,
+  columns: TableRecordValue[]|null = null): TableColValues {
+  if (params.sort) { applySort(values, params.sort, columns); }
   if (params.limit) { applyLimit(values, params.limit); }
   return values;
 }

app/server/lib/Export.ts

@@ -1,18 +1,19 @@
-import {buildColFilter} from 'app/common/ColumnFilterFunc';
+import { buildColFilter } from 'app/common/ColumnFilterFunc';
-import {RowRecord} from 'app/common/DocActions';
+import { RowRecord } from 'app/common/DocActions';
-import {DocData} from 'app/common/DocData';
+import { DocData } from 'app/common/DocData';
+import { DocumentSettings } from 'app/common/DocumentSettings';
 import * as gristTypes from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
-import {buildRowFilter} from 'app/common/RowFilterFunc';
+import { buildRowFilter } from 'app/common/RowFilterFunc';
-import {SchemaTypes} from 'app/common/schema';
+import { SchemaTypes } from 'app/common/schema';
-import {SortFunc} from 'app/common/SortFunc';
+import { SortFunc } from 'app/common/SortFunc';
-import {TableData} from 'app/common/TableData';
+import { Sort } from 'app/common/SortSpec';
-import {DocumentSettings} from 'app/common/DocumentSettings';
+import { TableData } from 'app/common/TableData';
-import {ActiveDoc} from 'app/server/lib/ActiveDoc';
+import { ActiveDoc } from 'app/server/lib/ActiveDoc';
-import {RequestWithLogin} from 'app/server/lib/Authorizer';
+import { RequestWithLogin } from 'app/server/lib/Authorizer';
-import {docSessionFromRequest} from 'app/server/lib/DocSession';
+import { docSessionFromRequest } from 'app/server/lib/DocSession';
-import {optIntegerParam, optJsonParam, stringParam} from 'app/server/lib/requestUtils';
+import { optIntegerParam, optJsonParam, stringParam } from 'app/server/lib/requestUtils';
-import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters';
+import { ServerColumnGetters } from 'app/server/lib/ServerColumnGetters';
 import * as express from 'express';
 import * as _ from 'underscore';
 
@@ -203,7 +204,7 @@ export async function exportTable(
 export async function exportSection(
   activeDoc: ActiveDoc,
   viewSectionId: number,
-  sortOrder: number[] | null,
+  sortSpec: Sort.SortSpec | null,
   filters: Filter[] | null,
   req: express.Request): Promise<ExportData> {
 
@@ -241,16 +242,16 @@ export async function exportSection(
     (field) => viewify(tableColsById[field.colRef], field));
 
   // The columns named in sort order need to now become display columns
-  sortOrder = sortOrder || gutil.safeJsonParse(viewSection.sortColRefs, []);
+  sortSpec = sortSpec || gutil.safeJsonParse(viewSection.sortColRefs, []);
   const fieldsByColRef = _.indexBy(fields, 'colRef');
-  sortOrder = sortOrder!.map((directionalColRef) => {
+  sortSpec = sortSpec!.map((colSpec) => {
-    const colRef = Math.abs(directionalColRef);
+    const colRef = Sort.getColRef(colSpec);
     const col = tableColsById[colRef];
     if (!col) {
       return 0;
     }
     const effectiveColRef = viewify(col, fieldsByColRef[colRef]).id;
-    return directionalColRef > 0 ? effectiveColRef : -effectiveColRef;
+    return Sort.swapColRef(colSpec, effectiveColRef);
   });
 
   // fetch actual data
@@ -260,7 +261,7 @@ export async function exportSection(
   // sort rows
   const getters = new ServerColumnGetters(rowIds, dataByColId, columns);
   const sorter = new SortFunc(getters);
-  sorter.updateSpec(sortOrder);
+  sorter.updateSpec(sortSpec);
   rowIds.sort((a, b) => sorter.compare(a, b));
   // create cell accessors
   const access = viewColumns.map(col => getters.getColGetter(col.id)!);

app/server/lib/ServerColumnGetters.ts

@@ -1,5 +1,8 @@
-import {ColumnGetters} from 'app/common/ColumnGetters';
+import { ColumnGetter, ColumnGetters } from 'app/common/ColumnGetters';
 import * as gristTypes from 'app/common/gristTypes';
+import { safeJsonParse } from 'app/common/gutil';
+import { choiceGetter } from 'app/common/SortFunc';
+import { Sort } from 'app/common/SortSpec';
 
 /**
  *
@@ -12,23 +15,33 @@ export class ServerColumnGetters implements ColumnGetters {
   private _colIndices: Map<number, string>;
 
   constructor(rowIds: number[], private _dataByColId: {[colId: string]: any}, private _columns: any[]) {
-    this._rowIndices = new Map<number, number>(rowIds.map((rowId, r) => [rowId, r] as [number, number]));
+    this._rowIndices = new Map<number, number>(rowIds.map((rowId, index) => [rowId, index] as [number, number]));
     this._colIndices = new Map<number, string>(_columns.map(col => [col.id, col.colId] as [number, string]));
   }
 
-  public getColGetter(colRef: number): ((rowId: number) => any) | null {
+  public getColGetter(colSpec: Sort.ColSpec): ColumnGetter | null {
+    const colRef = Sort.getColRef(colSpec);
     const colId = this._colIndices.get(colRef);
     if (colId === undefined) {
       return null;
     }
     const col = this._dataByColId[colId];
-    return rowId => {
+    let getter = (rowId: number) => {
       const idx = this._rowIndices.get(rowId);
       if (idx === undefined) {
         return null;
       }
       return col[idx];
     };
+    const details = Sort.specToDetails(colSpec);
+    if (details.orderByChoice) {
+      const rowModel = this._columns.find(c => c.id == colRef);
+      if (rowModel?.type === 'Choice') {
+        const choices: string[] = safeJsonParse(rowModel.widgetOptions, {}).choices || [];
+        getter = choiceGetter(getter, choices);
+      }
+    }
+    return getter;
   }
 
   public getManualSortGetter(): ((rowId: number) => any) | null {

sandbox/grist/sort_specs.py

@@ -0,0 +1,40 @@
+COL_SEPARATOR = ":"
+
+"""
+Helper module for sort expressions.
+Sort expressions are encoded as a positive number for ascending column,
+negative number for descending column. Can also be encoded as strings in a form:
+'-1:flag' or '1:flag;flag'
+Flags can be:
+- emptyLast to put empty values at the end.
+- orderByChoice: to order column by choice entry index rather then choice value.
+- naturalSort: to treat strings containing numbers as numbers and sort them accordingly.
+"""
+
+def col_ref(col_spec):
+  """
+  Gets column row id from column expression
+  """
+  return abs(col_spec if isinstance(col_spec, int) else int(col_spec.split(COL_SEPARATOR)[0]))
+
+def direction(col_spec):
+  """
+  Gets direction for column expression (1 for ascending - 1 for descending).
+  """
+  if isinstance(col_spec, int):
+    return 1 if col_spec >= 0 else -1
+  else:
+    assert col_spec
+    return 1 if col_spec[0] != "-" else -1
+
+def swap_col_ref(col_spec, new_col_ref):
+  """
+  Swaps colRef in colSpec preserving direction and options (used for display columns).
+  """
+  new_spec = direction(col_spec) * new_col_ref
+  if isinstance(col_spec, int):
+    return new_spec
+  else:
+    parts = col_spec.split(COL_SEPARATOR)
+    parts[0] = str(new_spec)
+    return COL_SEPARATOR.join(parts)

sandbox/grist/summary.py

@@ -5,6 +5,8 @@ import re
 import six
 
 from column import is_visible_column
+import sort_specs
+
 import logger
 log = logger.Logger(__name__, logger.INFO)
 
@@ -78,13 +80,14 @@ def _update_sort_spec(sort_spec, old_table, new_table):
   # When adjusting, we take a possibly negated old colRef, and produce a new colRef.
   # If anything is gone, we return 0, which will be excluded from the new sort spec.
   def adjust(col_spec):
-    sign = 1 if col_spec >= 0 else -1
+    old_colref = sort_specs.col_ref(col_spec)
-    return sign * new_cols_map.get(old_cols_map.get(abs(col_spec)), 0)
+    new_colref = new_cols_map.get(old_cols_map.get(old_colref), 0)
+    return sort_specs.swap_col_ref(col_spec, new_colref)
 
   try:
     old_sort_spec = json.loads(sort_spec)
     new_sort_spec = [adjust(col_spec) for col_spec in old_sort_spec]
-    new_sort_spec = [col_spec for col_spec in new_sort_spec if col_spec]
+    new_sort_spec = [col_spec for col_spec in new_sort_spec if sort_specs.col_ref(col_spec)]
     return json.dumps(new_sort_spec, separators=(',', ':'))
   except Exception:
     log.warn("update_summary_section: can't parse sortColRefs JSON; clearing sortColRefs")

sandbox/grist/test_sort_spec.py

@@ -0,0 +1,36 @@
+# coding=utf-8
+import unittest
+
+import sort_specs
+
+class TestSortSpec(unittest.TestCase):
+  def test_direction(self):
+    self.assertEqual(sort_specs.direction(1), 1)
+    self.assertEqual(sort_specs.direction(-1), -1)
+    self.assertEqual(sort_specs.direction('1'), 1)
+    self.assertEqual(sort_specs.direction('-1'), -1)
+    self.assertEqual(sort_specs.direction('1:emptyLast'), 1)
+    self.assertEqual(sort_specs.direction('1:emptyLast;orderByChoice'), 1)
+    self.assertEqual(sort_specs.direction('-1:emptyLast;orderByChoice'), -1)
+
+  def test_col_ref(self):
+    self.assertEqual(sort_specs.col_ref(1), 1)
+    self.assertEqual(sort_specs.col_ref(-1), 1)
+    self.assertEqual(sort_specs.col_ref('1'), 1)
+    self.assertEqual(sort_specs.col_ref('-1'), 1)
+    self.assertEqual(sort_specs.col_ref('1:emptyLast'), 1)
+    self.assertEqual(sort_specs.col_ref('1:emptyLast;orderByChoice'), 1)
+    self.assertEqual(sort_specs.col_ref('-1:emptyLast;orderByChoice'), 1)
+
+  def test_swap_col_ref(self):
+    self.assertEqual(sort_specs.swap_col_ref(1, 2), 2)
+    self.assertEqual(sort_specs.swap_col_ref(-1, 2), -2)
+    self.assertEqual(sort_specs.swap_col_ref('1', 2), '2')
+    self.assertEqual(sort_specs.swap_col_ref('-1', 2), '-2')
+    self.assertEqual(sort_specs.swap_col_ref('1:emptyLast', 2), '2:emptyLast')
+    self.assertEqual(
+      sort_specs.swap_col_ref('1:emptyLast;orderByChoice', 2),
+      '2:emptyLast;orderByChoice')
+    self.assertEqual(
+      sort_specs.swap_col_ref('-1:emptyLast;orderByChoice', 2),
+      '-2:emptyLast;orderByChoice')

sandbox/grist/useractions.py

@@ -11,6 +11,7 @@ import acl
 from acl_formula import parse_acl_formula_json
 import actions
 import column
+import sort_specs
 import identifiers
 from objtypes import strict_equal, encode_object
 import schema
@@ -877,7 +878,8 @@ class UserActions(object):
     for section in parent_sections:
       # Only iterates once for each section. Updated sort removes all columns being deleted.
       sort = json.loads(section.sortColRefs) if section.sortColRefs else []
-      updated_sort = [sort_ref for sort_ref in sort if abs(sort_ref) not in removed_col_refs]
+      updated_sort = [col_spec for col_spec in sort
+                      if sort_specs.col_ref(col_spec) not in removed_col_refs]
       if sort != updated_sort:
         re_sort_sections.append(section)
         re_sort_specs.append(json.dumps(updated_sort))

test/nbrowser/gristUtils.ts

@@ -922,18 +922,21 @@ export async function toggleSidePanel(which: 'right'|'left', goal: 'open'|'close
     return;
   }
 
+  // Adds '-ns' when narrow screen
+  const suffix = (await getWindowDimensions()).width < 768 ? '-ns' : '';
+
+  // click the opener and wait for the duration of the transition
+  await driver.find(`.test-${which}-opener${suffix}`).doClick();
+  await waitForSidePanel();
+}
+
+export async function waitForSidePanel() {
   // 0.4 is the duration of the transition setup in app/client/ui/PagePanels.ts for opening the
   // side panes
   const transitionDuration = 0.4;
 
   // let's add an extra delay of 0.1 for even more robustness
   const delta = 0.1;
-
-  // Adds '-ns' when narrow screen
-  const suffix = (await getWindowDimensions()).width < 768 ? '-ns' : '';
-
-  // click the opener and wait for the duration of the transition
-  await driver.find(`.test-${which}-opener${suffix}`).doClick();
   await driver.sleep((transitionDuration + delta) * 1000);
 }
 
@@ -1743,6 +1746,98 @@ export async function setRefTable(table: string) {
   await waitForServer();
 }
 
+// Add column to sort.
+export async function addColumnToSort(colName: RegExp|string) {
+  await driver.find(".test-vconfigtab-sort-add").click();
+  await driver.findContent(".test-vconfigtab-sort-add-menu-row", colName).click();
+  await driver.findContentWait(".test-vconfigtab-sort-row", colName, 100);
+}
+
+// Remove column from sort.
+export async function removeColumnFromSort(colName: RegExp|string) {
+  await findSortRow(colName).find(".test-vconfigtab-sort-remove").click();
+}
+
+// Toggle column sort order from ascending to descending, or vice-versa.
+export async function toggleSortOrder(colName: RegExp|string) {
+  await findSortRow(colName).find(".test-vconfigtab-sort-order").click();
+}
+
+// Change the column at the given sort position.
+export async function changeSortDropdown(colName: RegExp|string, newColName: RegExp|string) {
+  await findSortRow(colName).find(".test-select-row").click();
+  await driver.findContent("li .test-select-row", newColName).click();
+}
+
+// Reset the sort to the last saved sort.
+export async function revertSortConfig() {
+  await driver.find(".test-vconfigtab-sort-reset").click();
+}
+
+// Save the sort.
+export async function saveSortConfig() {
+  await driver.find(".test-vconfigtab-sort-save").click();
+  await waitForServer();
+}
+
+// Update the data positions to the given sort.
+export async function updateRowsBySort() {
+  await driver.find(".test-vconfigtab-sort-update").click();
+  await waitForServer(10000);
+}
+
+// Returns a WebElementPromise for the sort row of the given col name.
+export function findSortRow(colName: RegExp|string) {
+  return driver.findContent(".test-vconfigtab-sort-row", colName);
+}
+
+// Opens more sort options menu
+export async function openMoreSortOptions(colName: RegExp|string) {
+  const row = await findSortRow(colName);
+  return row.find(".test-vconfigtab-sort-options-icon").click();
+}
+
+// Selects one of the options in the more options menu.
+export async function toggleSortOption(option: SortOption) {
+  const label = await driver.find(`.test-vconfigtab-sort-option-${option} label`);
+  await label.click();
+  await waitForServer();
+}
+
+// Closes more sort options menu.
+export async function closeMoreSortOptionsMenu() {
+  await driver.sendKeys(Key.ESCAPE);
+}
+
+export type SortOption = "naturalSort" | "emptyLast" | "orderByChoice";
+export const SortOptions: ReadonlyArray<SortOption> = ["orderByChoice", "emptyLast", "naturalSort"];
+
+// Returns checked sort options for current column. Assumes the menu is opened.
+export async function getSortOptions(): Promise<SortOption[]> {
+  const options: SortOption[] = [];
+  for(const option of SortOptions) {
+    const list = await driver.findAll(`.test-vconfigtab-sort-option-${option} input:checked`);
+    if (list.length) {
+      options.push(option);
+    }
+  }
+  options.sort();
+  return options;
+}
+
+// Returns enabled entries in sort menu. Assumes the menu is opened.
+export async function getEnabledOptions(): Promise<SortOption[]> {
+  const options: SortOption[] = [];
+  for(const option of SortOptions) {
+    const list = await driver.findAll(`.test-vconfigtab-sort-option-${option}:not(.disabled)`);
+    if (list.length) {
+      options.push(option);
+    }
+  }
+  options.sort();
+  return options;
+}
+
 } // end of namespace gristUtils
 
 stackWrapOwnMethods(gristUtils);