import * as Types from "./DocApiTypes"; import { BulkColValues } from './GristData'; import { OpOptions, TableOperations, UpsertOptions } from './TableOperations'; import { arrayRepeat } from './gutil'; import flatMap = require('lodash/flatMap'); import isEqual = require('lodash/isEqual'); import pick = require('lodash/pick'); import groupBy = require('lodash/groupBy'); /** * An implementation of the TableOperations interface, given a platform * capable of applying user actions. Used by REST API server, and by the * Grist plugin API that is embedded in custom widgets. */ export class TableOperationsImpl implements TableOperations { public constructor(private _platform: TableOperationsPlatform, private _defaultOptions: OpOptions) { } public getTableId() { return this._platform.getTableId(); } public create(records: Types.NewRecord, options?: OpOptions): Promise; public create(records: Types.NewRecord[], options?: OpOptions): Promise; public async create(recordsOrRecord: Types.NewRecord[]|Types.NewRecord, options?: OpOptions): Promise { return await withRecords(recordsOrRecord, async (records) => { const postRecords = convertToBulkColValues(records); // postRecords can be an empty object, in that case we will create empty records. const ids = await this.addRecords(records.length, postRecords, options); return ids.map(id => ({id})); }); } public async update(recordOrRecords: Types.Record|Types.Record[], options?: OpOptions) { await withRecords(recordOrRecords, async (records) => { if (!areSameFields(records)) { this._platform.throwError('PATCH', 'requires all records to have same fields', 400); } const rowIds = records.map(r => r.id); const columnValues = convertToBulkColValues(records); if (!rowIds.length || !columnValues) { // For patch method, we require at least one valid record. this._platform.throwError('PATCH', 'requires a valid record object', 400); } await this.updateRecords(columnValues, rowIds, options); return []; }); } public async upsert(recordOrRecords: Types.AddOrUpdateRecord|Types.AddOrUpdateRecord[], upsertOptions?: UpsertOptions): Promise { await withRecords(recordOrRecords, async (records) => { const tableId = await this._platform.getTableId(); const options = { add: upsertOptions?.add, update: upsertOptions?.update, on_many: upsertOptions?.onMany, allow_empty_require: upsertOptions?.allowEmptyRequire }; const recordOptions: OpOptions = pick(upsertOptions, 'parseStrings'); // Group records based on having the same keys in `require` and `fields`. // A single bulk action will be applied to each group. // We don't want one bulk action for all records that might have different shapes, // because that would require filling arrays with null values. const recGroups = groupBy(records, rec => { const requireKeys = Object.keys(rec.require).sort().join(','); const fieldsKeys = Object.keys(rec.fields || {}).sort().join(','); return `${requireKeys}:${fieldsKeys}`; }); const actions = Object.values(recGroups).map(group => { const require = convertToBulkColValues(group.map(r => ({fields: r.require}))); const fields = convertToBulkColValues(group.map(r => ({fields: r.fields || {}}))); return ["BulkAddOrUpdateRecord", tableId, require, fields, options]; }); await this._applyUserActions(tableId, [...fieldNames(records)], actions, recordOptions); return []; }); } public async destroy(recordIdOrRecordIds: Types.RecordId|Types.RecordId[]): Promise { await withRecords(recordIdOrRecordIds, async (recordIds) => { const tableId = await this._platform.getTableId(); const actions = [['BulkRemoveRecord', tableId, recordIds]]; await this._applyUserActions(tableId, [], actions); return []; }); } // Update records identified by rowIds. Any invalid id fails // the request and returns a 400 error code. // This is exposed as a public method to support the older /data endpoint. public async updateRecords(columnValues: BulkColValues, rowIds: number[], options?: OpOptions) { await this._addOrUpdateRecords(columnValues, rowIds, 'BulkUpdateRecord', options); } /** * Adds records to a table. If columnValues is an empty object (or not provided) it will create empty records. * This is exposed as a public method to support the older /data endpoint. * @param columnValues Optional values for fields (can be an empty object to add empty records) * @param count Number of records to add */ public async addRecords( count: number, columnValues: BulkColValues, options?: OpOptions ): Promise { // user actions expect [null, ...] as row ids const rowIds = arrayRepeat(count, null); return this._addOrUpdateRecords(columnValues, rowIds, 'BulkAddRecord', options); } private async _addOrUpdateRecords( columnValues: BulkColValues, rowIds: (number | null)[], actionType: 'BulkUpdateRecord' | 'BulkAddRecord', options?: OpOptions ) { const tableId = await this._platform.getTableId(); const colNames = Object.keys(columnValues); const sandboxRes = await this._applyUserActions( tableId, colNames, [[actionType, tableId, rowIds, columnValues]], options ); return sandboxRes.retValues[0]; } // Apply the supplied actions with the given options. The tableId and // colNames are just to improve error reporting. private async _applyUserActions(tableId: string, colNames: string[], actions: any[][], options: OpOptions = {}): Promise { return handleSandboxErrorOnPlatform(tableId, colNames, this._platform.applyUserActions( actions, {...this._defaultOptions, ...options} ), this._platform); } } /** * The services needed by TableOperationsImpl. */ export interface TableOperationsPlatform { // Get the tableId of the table upon which we are supposed to operate. getTableId(): Promise; // Throw a platform-specific error. throwError(verb: string, text: string, status: number): never; // Apply the supplied actions with the given options. applyUserActions(actions: any[][], opts: any): Promise; } export function convertToBulkColValues(records: Array): BulkColValues { // User might want to create empty records, without providing a field name, for example for requests: // { records: [{}] }; { records: [{fields:{}}] } // Retrieve all field names from fields property. const result: BulkColValues = {}; for (const fieldName of fieldNames(records)) { result[fieldName] = records.map(record => record.fields?.[fieldName] ?? null); } return result; } export function fieldNames(records: any[]) { return new Set(flatMap(records, r => Object.keys({...r.fields, ...r.require}))); } export function areSameFields(records: Array) { const recordsFields = records.map(r => new Set(Object.keys(r.fields || {}))); return recordsFields.every(s => isEqual(recordsFields[0], s)); } /** * Adapt an operation that takes a list and returns a list to an input that may * be a single object or a list. If input is empty list, return the empty list. * If input is a single object, return a single object. Otherwise return a list. */ async function withRecords(recordsOrRecord: T[]|T, op: (records: T[]) => Promise): Promise { const records = Array.isArray(recordsOrRecord) ? recordsOrRecord : [recordsOrRecord]; const result = records.length == 0 ? [] : await op(records); return Array.isArray(recordsOrRecord) ? result : result[0]; } /** * Catches the errors thrown by the sandbox, and converts to more descriptive ones (such as for * invalid table names, columns, or rowIds) with better status codes. Accepts the table name, a * list of column names in that table, and a promise for the result of the sandbox call. */ export async function handleSandboxErrorOnPlatform( tableId: string, colNames: string[], p: Promise, platform: TableOperationsPlatform ): Promise { try { return await p; } catch (err) { const message = ((err instanceof Error) && err.message?.startsWith('[Sandbox] ')) ? err.message : undefined; if (message) { let match = message.match(/non-existent record #([0-9]+)/); if (match) { platform.throwError('', `Invalid row id ${match[1]}`, 400); } match = message.match( /\[Sandbox] (?:KeyError u?'(?:Table \w+ has no column )?|ValueError No such table: )(\w+)/ ); if (match) { if (match[1] === tableId) { platform.throwError('', `Table not found "${tableId}"`, 404); } else if (colNames.includes(match[1])) { platform.throwError('', `Invalid column "${match[1]}"`, 400); } } platform.throwError('', `Error manipulating data: ${message}`, 400); } throw err; } }