gristlabs_grist-core/app/common/TableData.ts

607 lines
21 KiB
TypeScript
Raw Normal View History

/**
* TableData maintains a single table's data.
*/
import {ActionDispatcher} from 'app/common/ActionDispatcher';
import {
BulkAddRecord, BulkColValues, CellValue, ColInfo, ColInfoWithId, ColValues, DocAction,
isSchemaAction, ReplaceTableData, RowRecord, TableDataAction} from 'app/common/DocActions';
import {getDefaultForType} from 'app/common/gristTypes';
(core) Guess date format during type conversion Summary: - Adds a dependency moment-guess (https://github.com/apoorv-mishra/moment-guess) to guess date formats from strings. However the npm package is missing source maps which leads to an ugly warning, so currently using a fork until https://github.com/apoorv-mishra/moment-guess/pull/22 is resolved. - Adds guessDateFormat using moment-guess to determine the best candidate date format. The logic may be refined for e.g. lossless imports where the stakes are higher, but for now we're just trying to make type conversions smoother. - Uses guessDateFormat to guess widget options when changing column type to date or datetime. - Uses the date format of the original column when possible instead of guessing. - Fixes a bug where choices were guessed based on the display column instead of the visible column, which made the guessed choices influenced by which values were referenced as well as completely broken when converting from reflist. - @dsagal @georgegevoian This builds on https://phab.getgrist.com/D3265, currently unmerged. That diff was created first to alert to the change. Without it there would still be similar test failures/changes here as the date format would often be concretely guessed and saved as YYYY-MM-DD instead of being left as the default `undefined` which is shows as YYYY-MM-DD in the dropdown. Test Plan: Added a unit test to `parseDate.ts`. Updated several browser tests which show the guessing in action during type conversion quite nicely. Reviewers: georgegevoian Reviewed By: georgegevoian Subscribers: dsagal, georgegevoian Differential Revision: https://phab.getgrist.com/D3264
2022-02-21 14:45:17 +00:00
import {arrayRemove, arraySplice, getDistinctValues} from 'app/common/gutil';
import {SchemaTypes} from "app/common/schema";
import isEqual = require('lodash/isEqual');
import fromPairs = require('lodash/fromPairs');
export interface ColTypeMap { [colId: string]: string; }
// This is the row ID used in the client, but it's helpful to have available in some common code
// as well, which is why it's declared in app/common. Note that for data actions and stored data,
// 'new' is not used.
export type UIRowId = number | 'new';
type UIRowFunc<T> = (rowId: UIRowId) => T;
interface ColData {
colId: string;
type: string;
defl: any;
values: CellValue[];
}
export interface SingleCell {
tableId: string;
colId: string;
rowId: number;
}
/**
* An interface for a table with rows that may be skipped.
*/
export interface SkippableRows {
// If there may be skippable rows, return a function to test rowIds for keeping.
getKeepFunc(): undefined | UIRowFunc<boolean>;
// Get a special row id which represents a skipped sequence of rows.
getSkipRowId(): number;
}
/**
* TableData class to maintain a single table's data.
*
* In the browser's memory, table data needs a representation that's reasonably compact. We
* represent it as column-wise arrays. (An early hope was to allow use of TypedArrays, but since
* types can be mixed, those are not used.)
*/
export class TableData extends ActionDispatcher implements SkippableRows {
private _tableId: string;
private _isLoaded: boolean = false;
private _fetchPromise?: Promise<void>;
// Storage of the underlying data. Each column is an array, all of the same length. Includes
// 'id' column, containing a reference to _rowIdCol.
private _columns: Map<string, ColData> = new Map();
// Array of all ColData objects, omitting 'id'.
private _colArray: ColData[] = [];
// The `id` column is direct reference to the 'id' column, and contains row ids.
private _rowIdCol: number[] = [];
// Maps row id to index in the arrays in _columns. I.e. it's the inverse of _rowIdCol.
private _rowMap: Map<number, number> = new Map();
constructor(tableId: string, tableData: TableDataAction|null, colTypes: ColTypeMap) {
super();
this._tableId = tableId;
// Initialize all columns to empty arrays, while nothing is yet loaded.
for (const colId in colTypes) {
if (colTypes.hasOwnProperty(colId)) {
const type = colTypes[colId];
const defl = getDefaultForType(type);
const colData: ColData = { colId, type, defl, values: [] };
this._columns.set(colId, colData);
this._colArray.push(colData);
}
}
this._columns.set('id', {colId: 'id', type: 'Id', defl: 0, values: this._rowIdCol});
if (tableData) {
this.loadData(tableData);
}
// TODO: We should probably unload big sets of data when no longer needed. This can be left for
// when we support loading only parts of a table.
}
/**
* Fetch data (as long as a fetch is not in progress), and load it in memory when done.
* Returns a promise that's resolved when data finishes loading, and isLoaded becomes true.
*/
public fetchData(fetchFunc: (tableId: string) => Promise<TableDataAction>): Promise<void> {
if (!this._fetchPromise) {
this._fetchPromise = fetchFunc(this._tableId).then(data => {
this._fetchPromise = undefined;
this.loadData(data);
});
}
return this._fetchPromise;
}
/**
* Populates the data for this table. Returns the array of old rowIds that were loaded before.
*/
public loadData(tableData: TableDataAction|ReplaceTableData): number[] {
const rowIds: number[] = tableData[2];
const colValues: BulkColValues = tableData[3];
const oldRowIds: number[] = this._rowIdCol.slice(0);
reassignArray(this._rowIdCol, rowIds);
for (const colData of this._colArray) {
const values = colData.colId === 'id' ? rowIds : colValues[colData.colId];
// If colId is missing from tableData, use an array of default values. Note that reusing
// default value like this is only OK because all default values we use are primitive.
reassignArray(colData.values, values || this._rowIdCol.map(() => colData.defl));
}
this._rowMap.clear();
for (let i = 0; i < rowIds.length; i++) {
this._rowMap.set(rowIds[i], i);
}
this._isLoaded = true;
return oldRowIds;
}
// Used by QuerySet to load new rows for onDemand tables.
public loadPartial(data: TableDataAction): void {
// Add the new rows, reusing BulkAddData code.
const rowIds: number[] = data[2];
this.onBulkAddRecord(data, data[1], rowIds, data[3]);
// Mark the table as loaded.
this._isLoaded = true;
}
// Used by QuerySet to remove unused rows for onDemand tables when a QuerySet is disposed.
public unloadPartial(rowIds: number[]): void {
// Remove the unneeded rows, reusing BulkRemoveRecord code.
this.onBulkRemoveRecord(['BulkRemoveRecord', this.tableId, rowIds], this.tableId, rowIds);
}
/**
* Read-only tableId.
*/
public get tableId(): string { return this._tableId; }
/**
* Boolean flag for whether the data for this table is already loaded.
*/
public get isLoaded(): boolean { return this._isLoaded; }
/**
* The number of records loaded in this table.
*/
public numRecords(): number { return this._rowIdCol.length; }
/**
* Returns the specified value from this table.
*/
public getValue(rowId: UIRowId, colId: string): CellValue|undefined {
const colData = this._columns.get(colId);
const index = this._rowMap.get(rowId as number); // rowId of 'new' will not be found.
return colData && index !== undefined ? colData.values[index] : undefined;
}
public hasRowId(rowId: number): boolean {
return this._rowMap.has(rowId);
}
/**
* Returns the index of the given rowId, if it exists, in the same unstable order that's
* returned by getRowIds() and getColValues().
*/
public getRowIdIndex(rowId: UIRowId): number|undefined {
return this._rowMap.get(rowId as number);
}
/**
* Given a column name, returns a function that takes a rowId and returns the value for that
* column of that row. The returned function is faster than getValue() calls.
*/
public getRowPropFunc(colId: string): UIRowFunc<CellValue|undefined> {
const colData = this._columns.get(colId);
if (!colData) { return () => undefined; }
const values = colData.values;
const rowMap = this._rowMap;
return (rowId: UIRowId) => values[rowMap.get(rowId as number)!];
}
// By default, no rows are skippable, all are kept.
public getKeepFunc(): undefined | UIRowFunc<boolean> {
return undefined;
}
// By default, no special row id for skip rows is needed.
public getSkipRowId(): number {
throw new Error('no skip row id defined');
}
/**
* Returns the list of all rowIds in this table, in unspecified and unstable order. Equivalent
* to getColValues('id').
*/
public getRowIds(): ReadonlyArray<number> {
return this._rowIdCol;
}
/**
* Sort and returns the list of all rowIds in this table.
*/
public getSortedRowIds(): number[] {
return this._rowIdCol.slice(0).sort((a, b) => a - b);
}
/**
* Returns true if cells may contain multiple versions (e.g. in diffs).
*/
public mayHaveVersions() {
return false;
}
/**
* Returns the list of colIds in this table, including 'id'.
*/
public getColIds(): string[] {
return Array.from(this._columns.keys());
}
/**
* Returns an unsorted list of all values in the given column. With no intervening actions,
* all arrays returned by getColValues() and getRowIds() are parallel to each other, i.e. the
* values at the same index correspond to the same record.
*/
public getColValues(colId: string): ReadonlyArray<CellValue>|undefined {
const colData = this._columns.get(colId);
return colData ? colData.values : undefined;
}
/**
* Returns a limited-sized set of distinct values from a column. If count is given, limits how many
* distinct values are returned.
*/
public getDistinctValues(colId: string, count: number = Infinity): Set<CellValue>|undefined {
const valColumn = this.getColValues(colId);
if (!valColumn) { return undefined; }
(core) Guess date format during type conversion Summary: - Adds a dependency moment-guess (https://github.com/apoorv-mishra/moment-guess) to guess date formats from strings. However the npm package is missing source maps which leads to an ugly warning, so currently using a fork until https://github.com/apoorv-mishra/moment-guess/pull/22 is resolved. - Adds guessDateFormat using moment-guess to determine the best candidate date format. The logic may be refined for e.g. lossless imports where the stakes are higher, but for now we're just trying to make type conversions smoother. - Uses guessDateFormat to guess widget options when changing column type to date or datetime. - Uses the date format of the original column when possible instead of guessing. - Fixes a bug where choices were guessed based on the display column instead of the visible column, which made the guessed choices influenced by which values were referenced as well as completely broken when converting from reflist. - @dsagal @georgegevoian This builds on https://phab.getgrist.com/D3265, currently unmerged. That diff was created first to alert to the change. Without it there would still be similar test failures/changes here as the date format would often be concretely guessed and saved as YYYY-MM-DD instead of being left as the default `undefined` which is shows as YYYY-MM-DD in the dropdown. Test Plan: Added a unit test to `parseDate.ts`. Updated several browser tests which show the guessing in action during type conversion quite nicely. Reviewers: georgegevoian Reviewed By: georgegevoian Subscribers: dsagal, georgegevoian Differential Revision: https://phab.getgrist.com/D3264
2022-02-21 14:45:17 +00:00
return getDistinctValues(valColumn, count);
}
/**
* Return data in TableDataAction form ['TableData', tableId, [...rowIds], {...}]
* Optionally takes a list of row ids to return data from. If a row id is
* not actually present in the table, a row of nulls will be returned for it.
*/
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
public getTableDataAction(desiredRowIds?: number[],
colIds?: string[]): TableDataAction {
colIds = colIds || this.getColIds();
const colIdSet = new Set<string>(colIds);
const rowIds = desiredRowIds || this.getRowIds();
let bulkColValues: {[colId: string]: CellValue[]};
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
const colArray = this._colArray.filter(({colId}) => colIdSet.has(colId));
if (desiredRowIds) {
const len = rowIds.length;
bulkColValues = {};
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
for (const colId of colIds) { bulkColValues[colId] = Array(len); }
for (let i = 0; i < len; i++) {
const index = this._rowMap.get(rowIds[i]);
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
for (const {colId, values} of colArray) {
const value = (index === undefined) ? null : values[index];
bulkColValues[colId][i] = value;
}
}
} else {
bulkColValues = fromPairs(
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
colIds
.filter(colId => colId !== 'id')
.map(colId => [colId, this.getColValues(colId)! as CellValue[]]));
}
return ['TableData',
this.tableId,
rowIds as number[],
bulkColValues];
}
public getBulkAddRecord(desiredRowIds?: number[]): BulkAddRecord {
const tableData = this.getTableDataAction(desiredRowIds?.sort((a, b) => a - b));
return [
'BulkAddRecord', tableData[1], tableData[2], tableData[3],
];
}
/**
* Returns the given columns type, if the column exists, or undefined otherwise.
*/
public getColType(colId: string): string|undefined {
const colData = this._columns.get(colId);
return colData ? colData.type : undefined;
}
/**
* Builds and returns a record object for the given rowId.
*/
public getRecord(rowId: number): undefined | RowRecord {
const index = this._rowMap.get(rowId);
if (index === undefined) { return undefined; }
const ret: RowRecord = { id: this._rowIdCol[index] };
for (const colData of this._colArray) {
ret[colData.colId] = colData.values[index];
}
return ret;
}
/**
* Builds and returns the list of all records on this table, in unspecified and unstable order.
*/
public getRecords(): RowRecord[] {
const records: RowRecord[] = this._rowIdCol.map((id) => ({ id }));
for (const {colId, values} of this._colArray) {
for (let i = 0; i < records.length; i++) {
records[i][colId] = values[i];
}
}
return records;
}
public filterRowIds(properties: {[key: string]: any}): number[] {
return this._filterRowIndices(properties).map(i => this._rowIdCol[i]);
}
/**
* Builds and returns the list of records in this table that match the given properties object.
* Properties may include 'id' and any table columns. Returned records are not sorted.
*/
public filterRecords(properties: {[key: string]: any}): RowRecord[] {
const rowIndices: number[] = this._filterRowIndices(properties);
// Convert the array of indices to an array of RowRecords.
const records: RowRecord[] = rowIndices.map(i => ({id: this._rowIdCol[i]}));
for (const {colId, values} of this._colArray) {
for (let i = 0; i < records.length; i++) {
records[i][colId] = values[rowIndices[i]];
}
}
return records;
}
/**
* Returns the rowId in the table where colValue is found in the column with the given colId.
*/
public findRow(colId: string, colValue: any): number {
const colData = this._columns.get(colId);
if (!colData) {
return 0;
}
const index = colData.values.indexOf(colValue);
return index < 0 ? 0 : this._rowIdCol[index];
}
/**
* Returns the first rowId matching the given filters, or 0 if no match. If there are multiple
* matches, it is unspecified which will be returned.
*/
public findMatchingRowId(properties: {[key: string]: CellValue | undefined}): number {
const props = Object.keys(properties).map(p => ({col: this._columns.get(p)!, value: properties[p]}));
if (!props.every((p) => p.col)) {
return 0;
}
return this._rowIdCol.find((id, i) =>
props.every((p) => isEqual(p.col.values[i], p.value))
) || 0;
}
/**
* Applies a DocAction received from the server; returns true, or false if it was skipped.
*/
public receiveAction(action: DocAction): boolean {
if (this._isLoaded || isSchemaAction(action)) {
this.dispatchAction(action);
return true;
}
return false;
}
// ---- The following methods implement ActionDispatcher interface ----
protected onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {
if (this._rowMap.get(rowId) !== undefined) {
// If adding a record that already exists, act like an update.
// We rely on this behavior for distributing attachment
// metadata.
this.onUpdateRecord(action, tableId, rowId, colValues);
return;
}
const index: number = this._rowIdCol.length;
this._rowMap.set(rowId, index);
this._rowIdCol[index] = rowId;
for (const {colId, defl, values} of this._colArray) {
values[index] = colValues.hasOwnProperty(colId) ? colValues[colId] : defl;
}
}
protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
let destIndex: number = this._rowIdCol.length;
for (let i = 0; i < rowIds.length; i++) {
const srcIndex = this._rowMap.get(rowIds[i]);
if (srcIndex !== undefined) {
// If adding a record that already exists, act like an update.
// We rely on this behavior for distributing attachment
// metadata.
for (const colId in colValues) {
if (colValues.hasOwnProperty(colId)) {
const colData = this._columns.get(colId);
if (colData) {
colData.values[srcIndex] = colValues[colId][i];
}
}
}
} else {
this._rowMap.set(rowIds[i], destIndex);
this._rowIdCol[destIndex] = rowIds[i];
for (const {colId, defl, values} of this._colArray) {
values[destIndex] = colValues.hasOwnProperty(colId) ? colValues[colId][i] : defl;
}
destIndex++;
}
}
}
protected onRemoveRecord(action: DocAction, tableId: string, rowId: number): void {
// Note that in this implementation, delete + undo will reorder the storage and the ordering
// of rows returned getRowIds() and similar methods.
const index = this._rowMap.get(rowId);
if (index !== undefined) {
const last: number = this._rowIdCol.length - 1;
// We keep the column-wise arrays dense by moving the last element into the freed-up spot.
for (const {values} of this._columns.values()) { // This adjusts _rowIdCol too.
values[index] = values[last];
values.pop();
}
this._rowMap.set(this._rowIdCol[index], index);
this._rowMap.delete(rowId);
}
}
protected onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {
const index = this._rowMap.get(rowId);
if (index !== undefined) {
for (const colId in colValues) {
if (colValues.hasOwnProperty(colId)) {
const colData = this._columns.get(colId);
if (colData) {
colData.values[index] = colValues[colId];
}
}
}
}
}
protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
for (let i = 0; i < rowIds.length; i++) {
const index = this._rowMap.get(rowIds[i]);
if (index !== undefined) {
for (const colId in colValues) {
if (colValues.hasOwnProperty(colId)) {
const colData = this._columns.get(colId);
if (colData) {
colData.values[index] = colValues[colId][i];
}
}
}
}
}
}
protected onReplaceTableData(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
this.loadData(action as ReplaceTableData);
}
protected onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void {
if (this._columns.has(colId)) { return; }
const type = colInfo.type;
const defl = getDefaultForType(type);
const colData: ColData = { colId, type, defl, values: this._rowIdCol.map(() => defl) };
this._columns.set(colId, colData);
this._colArray.push(colData);
}
protected onRemoveColumn(action: DocAction, tableId: string, colId: string): void {
const colData = this._columns.get(colId);
if (!colData) { return; }
this._columns.delete(colId);
arrayRemove(this._colArray, colData);
}
protected onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void {
const colData = this._columns.get(oldColId);
if (colData) {
colData.colId = newColId;
this._columns.set(newColId, colData);
this._columns.delete(oldColId);
}
}
protected onModifyColumn(action: DocAction, tableId: string, oldColId: string, colInfo: ColInfo): void {
const colData = this._columns.get(oldColId);
if (colData && colInfo.hasOwnProperty('type')) {
colData.type = colInfo.type;
colData.defl = getDefaultForType(colInfo.type);
}
}
protected onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void {
this._tableId = newTableId;
}
protected onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void {
// A table processing its own addition is a noop
}
protected onRemoveTable(action: DocAction, tableId: string): void {
// Stop dispatching actions if we've been deleted. We might also want to clean up in the future.
this._isLoaded = false;
}
private _filterRowIndices(properties: {[key: string]: any}): number[] {
const rowIndices: number[] = [];
// Array of {col: arrayOfColValues, value: valueToMatch}
const props = Object.keys(properties).map(p => ({col: this._columns.get(p)!, value: properties[p]}));
this._rowIdCol.forEach((id, i) => {
// Collect the indices of the matching rows.
if (props.every((p) => isEqual(p.col.values[i], p.value))) {
rowIndices.push(i);
}
});
return rowIndices;
}
}
// A type safe record of a meta table with types as defined in schema.ts
// '&' is used because declaring the id field and the index signature in one block gives a syntax error.
// The second part is basically equivalent to SchemaTypes[TableId]
// but TS sees that as incompatible with RowRecord and doesn't allow simple overrides in MetaTableData.
export type MetaRowRecord<TableId extends keyof SchemaTypes> =
{ id: number } &
{ [ColId in keyof SchemaTypes[TableId]]: SchemaTypes[TableId][ColId] & CellValue };
type MetaColId<TableId extends keyof SchemaTypes> = keyof MetaRowRecord<TableId> & string;
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
/**
* Behaves the same as TableData, but uses SchemaTypes for type safety of its columns.
*/
export class MetaTableData<TableId extends keyof SchemaTypes> extends TableData {
constructor(tableId: TableId, tableData: TableDataAction | null, colTypes: ColTypeMap) {
super(tableId, tableData, colTypes);
}
public getValue<ColId extends MetaColId<TableId>>(rowId: number, colId: ColId):
MetaRowRecord<TableId>[ColId] | undefined {
return super.getValue(rowId, colId) as any;
}
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
public getRecords(): Array<MetaRowRecord<TableId>> {
return super.getRecords() as any;
}
public getRecord(rowId: number): MetaRowRecord<TableId> | undefined {
return super.getRecord(rowId) as any;
}
public filterRecords(properties: Partial<MetaRowRecord<TableId>>): Array<MetaRowRecord<TableId>> {
return super.filterRecords(properties) as any;
}
public findMatchingRowId(properties: Partial<MetaRowRecord<TableId>>): number {
return super.findMatchingRowId(properties);
}
public getRowPropFunc<ColId extends MetaColId<TableId>>(
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
colId: ColId
): UIRowFunc<MetaRowRecord<TableId>[ColId]> {
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
return super.getRowPropFunc(colId as any) as any;
}
public getColValues<ColId extends MetaColId<TableId>>(
colId: ColId
): ReadonlyArray<MetaRowRecord<TableId>[ColId]> {
return super.getColValues(colId) as any;
}
public findRow<ColId extends MetaColId<TableId>>(
colId: ColId, colValue: MetaRowRecord<TableId>[ColId]
): number {
return super.findRow(colId, colValue);
}
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
}
function reassignArray<T>(targetArray: T[], sourceArray: T[]): void {
targetArray.length = 0;
arraySplice(targetArray, 0, sourceArray);
}