(core) updates from grist-core

app/client/components/GridView.js

@@ -316,6 +316,20 @@ GridView.gridCommands = {
     this._shiftSelect(-1, this.cellSelector.col.end, selector.ROW,
                       this.viewSection.viewFields().peekLength - 1);
   },
+  ctrlShiftDown: function () {
+    this._shiftSelectUntilContent(selector.COL, 1, this.cellSelector.row.end, this.getLastDataRowIndex());
+  },
+  ctrlShiftUp: function () {
+    this._shiftSelectUntilContent(selector.COL, -1, this.cellSelector.row.end, this.getLastDataRowIndex());
+  },
+  ctrlShiftRight: function () {
+    this._shiftSelectUntilContent(selector.ROW, 1, this.cellSelector.col.end,
+      this.viewSection.viewFields().peekLength - 1);
+  },
+  ctrlShiftLeft: function () {
+    this._shiftSelectUntilContent(selector.ROW, -1, this.cellSelector.col.end,
+      this.viewSection.viewFields().peekLength - 1);
+  },
   fillSelectionDown: function() { this.fillSelectionDown(); },
   selectAll: function() { this.selectAll(); },
 
@@ -403,6 +417,120 @@ GridView.prototype._shiftSelect = function(step, selectObs, exemptType, maxVal)
   selectObs(newVal);
 };
 
+GridView.prototype._shiftSelectUntilContent = function(type, direction, selectObs, maxVal) {
+  const selection = {
+    colStart: this.cellSelector.col.start(),
+    colEnd: this.cellSelector.col.end(),
+    rowStart: this.cellSelector.row.start(),
+    rowEnd: this.cellSelector.row.end(),
+  };
+
+  const steps = this._stepsToContent(type, direction, selection, maxVal);
+  if (steps > 0) { this._shiftSelect(direction * steps, selectObs, type, maxVal); }
+}
+
+GridView.prototype._stepsToContent = function (type, direction, selection, maxVal) {
+  const {colEnd: colEnd, rowEnd: rowEnd} = selection;
+  let selectionData;
+
+  const cursorCol = this.cursor.fieldIndex();
+  const cursorRow = this.cursor.rowIndex();
+
+  if (type === selector.ROW && direction > 0) {
+    if (colEnd + 1 > maxVal) { return 0; }
+
+    selectionData = this._selectionData({colStart: colEnd, colEnd: maxVal, rowStart: cursorRow, rowEnd: cursorRow});
+  } else if (type === selector.ROW && direction < 0) {
+    if (colEnd - 1 < 0) { return 0; }
+
+    selectionData = this._selectionData({colStart: 0, colEnd, rowStart: cursorRow, rowEnd: cursorRow});
+  } else if (type === selector.COL && direction > 0) {
+    if (rowEnd + 1 > maxVal) { return 0; }
+
+    selectionData = this._selectionData({colStart: cursorCol, colEnd: cursorCol, rowStart: rowEnd, rowEnd: maxVal});
+  } else if (type === selector.COL && direction < 0) {
+    if (rowEnd - 1 > maxVal) { return 0; }
+
+    selectionData = this._selectionData({colStart: cursorCol, colEnd: cursorCol, rowStart: 0, rowEnd});
+  }
+
+  const {fields, rowIndices} = selectionData;
+  if (type === selector.ROW && direction < 0) {
+    // When moving selection left, we step through fields in reverse order.
+    fields.reverse();
+  }
+  if (type === selector.COL && direction < 0) {
+    // When moving selection up, we step through rows in reverse order.
+    rowIndices.reverse();
+  }
+
+  const colValuesByIndex = {};
+  for (const field of fields) {
+    const displayColId = field.displayColModel.peek().colId.peek();
+    colValuesByIndex[field._index()] = this.tableModel.tableData.getColValues(displayColId);
+  }
+
+  let steps = 0;
+
+  if (type === selector.ROW) {
+    const rowIndex = rowIndices[0];
+    const isLastColEmpty = this._isCellValueEmpty(colValuesByIndex[colEnd][rowIndex]);
+    const isNextColEmpty = this._isCellValueEmpty(colValuesByIndex[colEnd + direction][rowIndex]);
+    const shouldStopOnEmptyValue = !isLastColEmpty && !isNextColEmpty;
+    for (let i = 1; i < fields.length; i++) {
+      const hasEmptyValues = this._isCellValueEmpty(colValuesByIndex[fields[i]._index()][rowIndex]);
+      if (hasEmptyValues && shouldStopOnEmptyValue) {
+        return steps;
+      } else if (!hasEmptyValues && !shouldStopOnEmptyValue) {
+        return steps + 1;
+      }
+
+      steps += 1;
+    }
+  } else {
+    const colValues = colValuesByIndex[fields[0]._index()];
+    const isLastRowEmpty = this._isCellValueEmpty(colValues[rowIndices[0]]);
+    const isNextRowEmpty = this._isCellValueEmpty(colValues[rowIndices[1]]);
+    const shouldStopOnEmptyValue = !isLastRowEmpty && !isNextRowEmpty;
+    for (let i = 1; i < rowIndices.length; i++) {
+      const hasEmptyValues = this._isCellValueEmpty(colValues[rowIndices[i]]);
+      if (hasEmptyValues && shouldStopOnEmptyValue) {
+        return steps;
+      } else if (!hasEmptyValues && !shouldStopOnEmptyValue) {
+        return steps + 1;
+      }
+
+      steps += 1;
+    }
+  }
+
+  return steps;
+}
+
+GridView.prototype._selectionData = function({colStart, colEnd, rowStart, rowEnd}) {
+  const fields = [];
+  for (let i = colStart; i <= colEnd; i++) {
+    const field = this.viewSection.viewFields().at(i);
+    if (!field) { continue; }
+
+    fields.push(field);
+  }
+
+  const rowIndices = [];
+  for (let i = rowStart; i <= rowEnd; i++) {
+    const rowId = this.viewData.getRowId(i);
+    if (!rowId) { continue; }
+
+    rowIndices.push(this.tableModel.tableData.getRowIdIndex(rowId));
+  }
+
+  return {fields, rowIndices};
+}
+
+GridView.prototype._isCellValueEmpty = function(value) {
+  return value === null || value === undefined || value === '' || value === 'false';
+}
+
 /**
  * Pastes the provided data at the current cursor.
  *
@@ -532,15 +660,15 @@ GridView.prototype.fillSelectionDown = function() {
 
 
 /**
- * Returns a GridSelection of the selected rows and cols
+ * Returns a CopySelection of the selected rows and cols
  * @returns {Object} CopySelection
  */
 GridView.prototype.getSelection = function() {
-  var rowIds = [], fields = [], rowStyle = {}, colStyle = {};
-  var colStart = this.cellSelector.colLower();
-  var colEnd = this.cellSelector.colUpper();
-  var rowStart = this.cellSelector.rowLower();
-  var rowEnd = this.cellSelector.rowUpper();
+  let rowIds = [], fields = [], rowStyle = {}, colStyle = {};
+  let colStart = this.cellSelector.colLower();
+  let colEnd = this.cellSelector.colUpper();
+  let rowStart = this.cellSelector.rowLower();
+  let rowEnd = this.cellSelector.rowUpper();
 
   // If there is no selection, just copy/paste the cursor cell
   if (this.cellSelector.isCurrentSelectType(selector.NONE)) {

app/client/components/commandList.ts

@@ -50,6 +50,10 @@ export type CommandName =
   | 'shiftUp'
   | 'shiftRight'
   | 'shiftLeft'
+  | 'ctrlShiftDown'
+  | 'ctrlShiftUp'
+  | 'ctrlShiftRight'
+  | 'ctrlShiftLeft'
   | 'selectAll'
   | 'copyLink'
   | 'editField'
@@ -373,6 +377,22 @@ export const groups: CommendGroupDef[] = [{
      name: 'shiftLeft',
      keys: ['Shift+Left'],
      desc: 'Adds the element to the left of the cursor to the selected range'
+    }, {
+      name: 'ctrlShiftDown',
+      keys: ['Mod+Shift+Down'],
+      desc: 'Adds all elements below the cursor to the selected range'
+    }, {
+      name: 'ctrlShiftUp',
+      keys: ['Mod+Shift+Up'],
+      desc: 'Adds all elements above the cursor to the selected range'
+    }, {
+      name: 'ctrlShiftRight',
+      keys: ['Mod+Shift+Right'],
+      desc: 'Adds all elements to the right of the cursor to the selected range'
+    }, {
+      name: 'ctrlShiftLeft',
+      keys: ['Mod+Shift+Left'],
+      desc: 'Adds all elements to the left of the cursor to the selected range'
     }, {
       name: 'selectAll',
       keys: ['Mod+A'],

app/plugin/DocApiTypes-ti.ts

@@ -68,6 +68,10 @@ export const ColumnsPatch = t.iface([], {
   "columns": t.tuple("RecordWithStringId", t.rest(t.array("RecordWithStringId"))),
 });
 
+export const ColumnsPut = t.iface([], {
+  "columns": t.tuple("RecordWithStringId", t.rest(t.array("RecordWithStringId"))),
+});
+
 export const TablePost = t.iface(["ColumnsPost"], {
   "id": t.opt("string"),
 });
@@ -93,6 +97,7 @@ const exportedTypeSuite: t.ITypeSuite = {
   MinimalRecord,
   ColumnsPost,
   ColumnsPatch,
+  ColumnsPut,
   TablePost,
   TablesPost,
   TablesPatch,

app/plugin/DocApiTypes.ts

@@ -88,6 +88,10 @@ export interface ColumnsPatch {
   columns: [RecordWithStringId, ...RecordWithStringId[]]; // at least one column is required
 }
 
+export interface ColumnsPut {
+  columns: [RecordWithStringId, ...RecordWithStringId[]]; // at least one column is required
+}
+
 /**
  * Creating tables requires a list of columns.
  * `fields` is not accepted because it's not generally sensible to set the metadata fields on new tables.

app/server/lib/DocApi.ts

@@ -2,7 +2,14 @@ import {concatenateSummaries, summarizeAction} from "app/common/ActionSummarizer
 import {createEmptyActionSummary} from "app/common/ActionSummary";
 import {ApiError, LimitType} from 'app/common/ApiError';
 import {BrowserSettings} from "app/common/BrowserSettings";
-import {BulkColValues, ColValues, fromTableDataAction, TableColValues, TableRecordValue} from 'app/common/DocActions';
+import {
+  BulkColValues,
+  ColValues,
+  fromTableDataAction,
+  TableColValues,
+  TableRecordValue,
+  UserAction
+} from 'app/common/DocActions';
 import {isRaisedException} from "app/common/gristTypes";
 import {buildUrlId, parseUrlId} from "app/common/gristUrls";
 import {isAffirmative} from "app/common/gutil";
@@ -94,7 +101,7 @@ type WithDocHandler = (activeDoc: ActiveDoc, req: RequestWithLogin, resp: Respon
 // Schema validators for api endpoints that creates or updates records.
 const {
   RecordsPatch, RecordsPost, RecordsPut,
-  ColumnsPost, ColumnsPatch,
+  ColumnsPost, ColumnsPatch, ColumnsPut,
   TablesPost, TablesPatch,
 } = t.createCheckers(DocApiTypesTI, GristDataTI);
 
@@ -194,16 +201,23 @@ export class DocWorkerApi {
     }
 
     async function getTableRecords(
-      activeDoc: ActiveDoc, req: RequestWithLogin, optTableId?: string
+      activeDoc: ActiveDoc, req: RequestWithLogin, opts?: { optTableId?: string; includeHidden?: boolean }
     ): Promise<TableRecordValue[]> {
-      const columnData = await getTableData(activeDoc, req, optTableId);
-      const fieldNames = Object.keys(columnData)
-        .filter(k => !(
-          ["id", "manualSort"].includes(k)
-          || k.startsWith("gristHelper_")
-        ));
+      const columnData = await getTableData(activeDoc, req, opts?.optTableId);
+      const fieldNames = Object.keys(columnData).filter((k) => {
+        if (k === "id") {
+          return false;
+        }
+        if (
+          !opts?.includeHidden &&
+          (k === "manualSort" || k.startsWith("gristHelper_"))
+        ) {
+          return false;
+        }
+        return true;
+      });
       return columnData.id.map((id, index) => {
-        const result: TableRecordValue = {id, fields: {}};
+        const result: TableRecordValue = { id, fields: {} };
         for (const key of fieldNames) {
           let value = columnData[key][index];
           if (isRaisedException(value)) {
@@ -226,7 +240,9 @@ export class DocWorkerApi {
     // Get the specified table in record-oriented format
     this._app.get('/api/docs/:docId/tables/:tableId/records', canView,
       withDoc(async (activeDoc, req, res) => {
-        const records = await getTableRecords(activeDoc, req);
+        const records = await getTableRecords(activeDoc, req,
+          { includeHidden: isAffirmative(req.query.hidden) }
+        );
         res.json({records});
       })
     );
@@ -357,8 +373,9 @@ export class DocWorkerApi {
     this._app.get('/api/docs/:docId/tables/:tableId/columns', canView,
       withDoc(async (activeDoc, req, res) => {
         const tableId = req.params.tableId;
+        const includeHidden = isAffirmative(req.query.hidden);
         const columns = await handleSandboxError('', [],
-          activeDoc.getTableCols(docSessionFromRequest(req), tableId));
+          activeDoc.getTableCols(docSessionFromRequest(req), tableId, includeHidden));
         res.json({columns});
       })
     );
@@ -366,7 +383,7 @@ export class DocWorkerApi {
     // Get the tables of the specified document in recordish format
     this._app.get('/api/docs/:docId/tables', canView,
       withDoc(async (activeDoc, req, res) => {
-        const records = await getTableRecords(activeDoc, req, "_grist_Tables");
+        const records = await getTableRecords(activeDoc, req, { optTableId: "_grist_Tables" });
         const tables = records.map((record) => ({
           id: record.fields.tableId,
           fields: {
@@ -395,7 +412,7 @@ export class DocWorkerApi {
 
     // Returns cleaned metadata for all attachments in /records format.
     this._app.get('/api/docs/:docId/attachments', canView, withDoc(async (activeDoc, req, res) => {
-      const rawRecords = await getTableRecords(activeDoc, req, "_grist_Attachments");
+      const rawRecords = await getTableRecords(activeDoc, req, { optTableId: "_grist_Attachments" });
       const records = rawRecords.map(r => ({
         id: r.id,
         fields: cleanAttachmentRecord(r.fields as MetaRowRecord<"_grist_Attachments">),
@@ -681,6 +698,54 @@ export class DocWorkerApi {
       })
     );
 
+    // Add or update records given in records format
+    this._app.put('/api/docs/:docId/tables/:tableId/columns', canEdit, validate(ColumnsPut),
+      withDoc(async (activeDoc, req, res) => {
+        const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables");
+        const columnsTable = activeDoc.docData!.getMetaTable("_grist_Tables_column");
+        const {tableId} = req.params;
+        const tableRef = tablesTable.findMatchingRowId({tableId});
+        if (!tableRef) {
+          throw new ApiError(`Table not found "${tableId}"`, 404);
+        }
+        const body = req.body as Types.ColumnsPut;
+
+        const addActions: UserAction[] = [];
+        const updateActions: UserAction[] = [];
+        const updatedColumnsIds = new Set();
+
+        for (const col of body.columns) {
+          const id = columnsTable.findMatchingRowId({parentId: tableRef, colId: col.id});
+          if (id) {
+            updateActions.push( ['UpdateRecord', '_grist_Tables_column', id, col.fields] );
+            updatedColumnsIds.add( id );
+          } else {
+            addActions.push( ['AddVisibleColumn', tableId, col.id, col.fields] );
+          }
+        }
+
+        const getRemoveAction = async () => {
+          const columns = await handleSandboxError('', [],
+            activeDoc.getTableCols(docSessionFromRequest(req), tableId));
+          const columnsToRemove = columns
+            .map(col => col.fields.colRef as number)
+            .filter(colRef => !updatedColumnsIds.has(colRef));
+
+          return [ 'BulkRemoveRecord', '_grist_Tables_column', columnsToRemove ];
+        };
+
+        const actions = [
+          ...(!isAffirmative(req.query.noupdate) ? updateActions : []),
+          ...(!isAffirmative(req.query.noadd) ? addActions : []),
+          ...(isAffirmative(req.query.replaceall) ? [ await getRemoveAction() ] : [] )
+        ];
+        await handleSandboxError(tableId, [],
+          activeDoc.applyUserActions(docSessionFromRequest(req), actions)
+        );
+        res.json(null);
+      })
+    );
+
     // Add a new webhook and trigger
     this._app.post('/api/docs/:docId/webhooks', isOwner, validate(WebhookSubscribeCollection),
       withDoc(async (activeDoc, req, res) => {