(core) control the distribution of attachment metadata

Summary:
for users who don't automatically have deep rights
to the document, provide them with attachment metadata only
for rows they have access to. This is a little tricky to
do efficiently. We provide attachment metadata when an
individual table is fetched, rather than on initial document
load, so we don't block that load on a full document scan.
We provide attachment metadata to a client when we see that
we are shipping rows mentioning particular attachments,
without making any effort to keep track of the metadata they
already have.

Test Plan: updated tests

Reviewers: dsagal, jarek

Reviewed By: dsagal, jarek

Differential Revision: https://phab.getgrist.com/D3722

app/common/ActiveDocAPI.ts

@@ -1,5 +1,5 @@
 import {ActionGroup} from 'app/common/ActionGroup';
-import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions';
+import {BulkAddRecord, CellValue, TableDataAction, UserAction} from 'app/common/DocActions';
 import {FormulaProperties} from 'app/common/GranularAccessClause';
 import {UIRowId} from 'app/common/UIRowId';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
@@ -137,14 +137,30 @@ export interface QueryFilters {
 // - empty: value should be falsy (e.g. null) or an empty list, filters is ignored
 export type QueryOperation = "in" | "intersects" | "empty";
 
+/**
+ * Results of fetching a table. Includes the table data you would
+ * expect. May now also include attachment metadata referred to in the table
+ * data. Attachment data is expressed as a BulkAddRecord, since it is
+ * not a complete table, just selected rows. Attachment data is
+ * currently included in fetches when (1) granular access control is
+ * in effect, and (2) the user is neither an owner nor someone with
+ * read access to the entire document, and (3) there is an attachment
+ * column in the fetched table. This is exactly what the standard
+ * Grist client needs, but in future it might be desirable to give
+ * more control over this behavior.
+ */
+export interface TableFetchResult {
+  tableData: TableDataAction;
+  attachments?: BulkAddRecord;
+}
+
 /**
  * Response from useQuerySet(). A query returns data AND creates a subscription to receive
  * DocActions that affect this data. The querySubId field identifies this subscription, and must
  * be used in a disposeQuerySet() call to unsubscribe.
  */
-export interface QueryResult {
+export interface QueryResult extends TableFetchResult {
   querySubId: number;     // ID of the subscription, to use with disposeQuerySet.
-  tableData: TableDataAction;
 }
 
 /**
@@ -229,7 +245,7 @@ export interface ActiveDocAPI {
   /**
    * Fetches a particular table from the data engine to return to the client.
    */
-  fetchTable(tableId: string): Promise<TableDataAction>;
+  fetchTable(tableId: string): Promise<TableFetchResult>;
 
   /**
    * Fetches the generated Python code for this document. (TODO rename this misnomer.)

app/common/AttachmentColumns.ts

@@ -0,0 +1,82 @@
+import { AddRecord, BulkAddRecord, BulkRemoveRecord, BulkUpdateRecord,
+         getColIdsFromDocAction, getColValuesFromDocAction,
+         getTableId, RemoveRecord, ReplaceTableData, TableDataAction,
+         UpdateRecord } from 'app/common/DocActions';
+import { DocData } from 'app/common/DocData';
+import { isNumber } from 'app/common/gutil';
+
+/**
+ * Represent current attachment columns as a map from tableId to a set of
+ * colIds.
+ */
+export type AttachmentColumns = Map<string, Set<string>>;
+
+/**
+ * Enumerate attachment columns, represented as a map from tableId to
+ * a set of colIds.
+ */
+export function getAttachmentColumns(metaDocData: DocData): AttachmentColumns {
+  const tablesTable = metaDocData.getMetaTable('_grist_Tables');
+  const columnsTable = metaDocData.getMetaTable('_grist_Tables_column');
+  const attachmentColumns: Map<string, Set<string>> = new Map();
+  for (const column of columnsTable.filterRecords({type: 'Attachments'})) {
+    const table = tablesTable.getRecord(column.parentId);
+    const tableId = table?.tableId;
+    if (!tableId) {
+      /* should never happen */
+      throw new Error('table not found');
+    }
+    if (!attachmentColumns.has(tableId)) {
+      attachmentColumns.set(tableId, new Set());
+    }
+    attachmentColumns.get(tableId)!.add(column.colId);
+  }
+  return attachmentColumns;
+}
+
+/**
+ * Get IDs of attachments that are present in attachment columns in an action.
+ */
+export function gatherAttachmentIds(
+  attachmentColumns: AttachmentColumns,
+  action: AddRecord | BulkAddRecord | UpdateRecord | BulkUpdateRecord |
+    RemoveRecord | BulkRemoveRecord | ReplaceTableData | TableDataAction
+): Set<number> {
+  const tableId = getTableId(action);
+  const attColumns = attachmentColumns.get(tableId);
+  const colIds = getColIdsFromDocAction(action) || [];
+  const attIds = new Set<number>();
+  if (!attColumns || !colIds.some(colId => attColumns.has(colId))) {
+    return attIds;
+  }
+  for (const colId of colIds) {
+    if (!attColumns.has(colId)) { continue; }
+    const values = getColValuesFromDocAction(action, colId);
+    if (!values) { continue; }
+    for (const v of values) {
+      // We expect an array. What should we do with other types?
+      // If we were confident no part of Grist would interpret non-array
+      // values as attachment ids, then we should let them be added, as
+      // part of Grist's spreadsheet-style willingness to allow invalid
+      // data. I decided to go ahead and require that numbers or number-like
+      // strings should be checked as if they were attachment ids, just in
+      // case. But if this proves awkward for someone, it could be reasonable
+      // to only check ids in an array after confirming Grist is strict in
+      // how it interprets material in attachment cells.
+      if (typeof v === 'number') {
+        attIds.add(v);
+      } else if (Array.isArray(v)) {
+        for (const p of v) {
+          if (typeof p === 'number') {
+            attIds.add(p);
+          }
+        }
+      } else if (typeof v === 'boolean' || v === null) {
+        // Nothing obvious to do here.
+      } else if (isNumber(v)) {
+        attIds.add(Math.round(parseFloat(v)));
+      }
+    }
+  }
+  return attIds;
+}

app/common/DocActions.ts

@@ -178,9 +178,12 @@ export function toTableDataAction(tableId: string, colValues: TableColValues): T
 
 // Convert from TableDataAction (used mainly by the sandbox) to TableColValues (used by DocStorage
 // and external APIs).
-export function fromTableDataAction(tableData: TableDataAction): TableColValues {
-  const rowIds: number[] = tableData[2];
-  const colValues: BulkColValues = tableData[3];
+// Also accepts a TableDataAction nested as a tableData member of a larger structure,
+// for convenience in dealing with the result of fetches.
+export function fromTableDataAction(tableData: TableDataAction|{tableData: TableDataAction}): TableColValues {
+  const data = ('tableData' in tableData) ? tableData.tableData : tableData;
+  const rowIds: number[] = data[2];
+  const colValues: BulkColValues = data[3];
   return {id: rowIds, ...colValues};
 }
 
@@ -203,3 +206,33 @@ export function getColValues(records: Partial<RowRecord>[]): BulkColValues {
   }
   return result;
 }
+
+/**
+ * Extract the col ids mentioned in a record-related DocAction as a list
+ * (even if the action is not a bulk action). Returns undefined if no col ids
+ * mentioned.
+ */
+export function getColIdsFromDocAction(docActions: RemoveRecord | BulkRemoveRecord | AddRecord |
+  BulkAddRecord | UpdateRecord | BulkUpdateRecord | ReplaceTableData |
+  TableDataAction): string[] | undefined {
+  if (docActions[3]) { return Object.keys(docActions[3]); }
+  return undefined;
+}
+
+/**
+ * Extract column values for a particular column as CellValue[] from a
+ * record-related DocAction. Undefined if absent.
+ */
+export function getColValuesFromDocAction(docAction: RemoveRecord | BulkRemoveRecord | AddRecord |
+  BulkAddRecord | UpdateRecord | BulkUpdateRecord | ReplaceTableData |
+  TableDataAction, colId: string): CellValue[]|undefined {
+  const colValues = docAction[3];
+  if (!colValues) { return undefined; }
+  const cellValues = colValues[colId];
+  if (!cellValues) { return undefined; }
+  if (Array.isArray(docAction[2])) {
+    return cellValues as CellValue[];
+  } else {
+    return [cellValues as CellValue];
+  }
+}

app/common/DocData.ts

@@ -9,24 +9,39 @@ import {schema, SchemaTypes} from 'app/common/schema';
 import fromPairs = require('lodash/fromPairs');
 import groupBy = require('lodash/groupBy');
 import {ActionDispatcher} from './ActionDispatcher';
+import {TableFetchResult} from './ActiveDocAPI';
 import {
   BulkColValues, ColInfo, ColInfoWithId, ColValues, DocAction,
   RowRecord, TableDataAction
 } from './DocActions';
 import {ColTypeMap, MetaRowRecord, MetaTableData, TableData} from './TableData';
 
-type FetchTableFunc = (tableId: string) => Promise<TableDataAction>;
+type FetchTableFunc = (tableId: string) => Promise<TableFetchResult>;
 
 export class DocData extends ActionDispatcher {
   private _tables: Map<string, TableData> = new Map();
 
+  private _fetchTableFunc: (tableId: string) => Promise<TableDataAction>;
+
   /**
    * If metaTableData is not supplied, then any tables needed should be loaded manually,
    * using syncTable(). All column types will be set to Any, which will affect default
    * values.
    */
-  constructor(private _fetchTableFunc: FetchTableFunc, metaTableData: {[tableId: string]: TableDataAction} | null) {
+  constructor(fetchTableFunc: FetchTableFunc, metaTableData: {[tableId: string]: TableDataAction} | null) {
     super();
+    // Wrap fetchTableFunc slightly to handle any extra attachment data that
+    // may come along for the ride.
+    this._fetchTableFunc = async (tableId: string) => {
+      const {tableData, attachments} = await fetchTableFunc(tableId);
+      if (attachments) {
+        // Back-end doesn't keep track of which attachments we already have,
+        // so there may be duplicates of rows we already have - but happily
+        // BulkAddRecord overwrites duplicates now.
+        this.receiveAction(attachments);
+      }
+      return tableData;
+    };
     if (metaTableData === null) { return; }
     // Create all meta tables, and populate data we already have.
     for (const tableId in schema) {

app/common/TableData.ts

@@ -2,7 +2,8 @@
  * TableData maintains a single table's data.
  */
 import {ActionDispatcher} from 'app/common/ActionDispatcher';
-import {BulkColValues, CellValue, ColInfo, ColInfoWithId, ColValues, DocAction,
+import {
+  BulkAddRecord, BulkColValues, CellValue, ColInfo, ColInfoWithId, ColValues, DocAction,
         isSchemaAction, ReplaceTableData, RowRecord, TableDataAction} from 'app/common/DocActions';
 import {getDefaultForType} from 'app/common/gristTypes';
 import {arrayRemove, arraySplice, getDistinctValues} from 'app/common/gutil';
@@ -250,16 +251,40 @@ export class TableData extends ActionDispatcher implements SkippableRows {
 
   /**
    * Return data in TableDataAction form ['TableData', tableId, [...rowIds], {...}]
+   * Optionally takes a list of row ids to return data from. If a row id is
+   * not actually present in the table, a row of nulls will be returned for it.
    */
-  public getTableDataAction(): TableDataAction {
-    const rowIds = this.getRowIds();
+  public getTableDataAction(desiredRowIds?: number[]): TableDataAction {
+    const rowIds = desiredRowIds || this.getRowIds();
+    let bulkColValues: {[colId: string]: CellValue[]};
+    if (desiredRowIds) {
+      const len = rowIds.length;
+      bulkColValues = {};
+      for (const colId of this.getColIds()) { bulkColValues[colId] = Array(len); }
+      for (let i = 0; i < len; i++) {
+        const index = this._rowMap.get(rowIds[i]);
+        for (const {colId, values} of this._colArray) {
+          const value = (index === undefined) ? null : values[index];
+          bulkColValues[colId][i] = value;
+        }
+      }
+    } else {
+      bulkColValues = fromPairs(
+        this.getColIds()
+          .filter(colId => colId !== 'id')
+          .map(colId => [colId, this.getColValues(colId)! as CellValue[]]));
+    }
     return ['TableData',
             this.tableId,
             rowIds as number[],
-            fromPairs(
-              this.getColIds()
-                .filter(colId => colId !== 'id')
-                .map(colId => [colId, this.getColValues(colId)! as CellValue[]]))];
+            bulkColValues];
+  }
+
+  public getBulkAddRecord(desiredRowIds?: number[]): BulkAddRecord {
+    const tableData = this.getTableDataAction(desiredRowIds?.sort((a, b) => a - b));
+    return [
+      'BulkAddRecord', tableData[1], tableData[2], tableData[3],
+    ];
   }
 
   /**
@@ -357,6 +382,13 @@ export class TableData extends ActionDispatcher implements SkippableRows {
   // ---- The following methods implement ActionDispatcher interface ----
 
   protected onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {
+    if (this._rowMap.get(rowId) !== undefined) {
+      // If adding a record that already exists, act like an update.
+      // We rely on this behavior for distributing attachment
+      // metadata.
+      this.onUpdateRecord(action, tableId, rowId, colValues);
+      return;
+    }
     const index: number = this._rowIdCol.length;
     this._rowMap.set(rowId, index);
     this._rowIdCol[index] = rowId;
@@ -366,14 +398,28 @@ export class TableData extends ActionDispatcher implements SkippableRows {
   }
 
   protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
-    const index: number = this._rowIdCol.length;
+    let destIndex: number = this._rowIdCol.length;
     for (let i = 0; i < rowIds.length; i++) {
-      this._rowMap.set(rowIds[i], index + i);
-      this._rowIdCol[index + i] = rowIds[i];
-    }
-    for (const {colId, defl, values} of this._colArray) {
-      for (let i = 0; i < rowIds.length; i++) {
-        values[index + i] = colValues.hasOwnProperty(colId) ? colValues[colId][i] : defl;
+      const srcIndex = this._rowMap.get(rowIds[i]);
+      if (srcIndex !== undefined) {
+        // If adding a record that already exists, act like an update.
+        // We rely on this behavior for distributing attachment
+        // metadata.
+        for (const colId in colValues) {
+          if (colValues.hasOwnProperty(colId)) {
+            const colData = this._columns.get(colId);
+            if (colData) {
+              colData.values[srcIndex] = colValues[colId][i];
+            }
+          }
+        }
+      } else {
+        this._rowMap.set(rowIds[i], destIndex);
+        this._rowIdCol[destIndex] = rowIds[i];
+        for (const {colId, defl, values} of this._colArray) {
+          values[destIndex] = colValues.hasOwnProperty(colId) ? colValues[colId][i] : defl;
+        }
+        destIndex++;
       }
     }
   }