2020-07-21 13:20:51 +00:00
|
|
|
/**
|
|
|
|
* This mirrors action definitions from sandbox/grist/actions.py
|
|
|
|
*/
|
|
|
|
|
2020-08-13 18:13:16 +00:00
|
|
|
// Some definitions have moved to be part of plugin API.
|
2022-03-15 14:35:15 +00:00
|
|
|
import { BulkColValues, CellValue, RowRecord } from 'app/plugin/GristData';
|
(core) Speed up and upgrade build.
Summary:
- Upgrades to build-related packages:
- Upgrade typescript, related libraries and typings.
- Upgrade webpack, eslint; add tsc-watch, node-dev, eslint_d.
- Build organization changes:
- Build webpack from original typescript, transpiling only; with errors still
reported by a background tsc watching process.
- Typescript-related changes:
- Reduce imports of AWS dependencies (very noticeable speedup)
- Avoid auto-loading global @types
- Client code is now built with isolatedModules flag (for safe transpilation)
- Use allowJs to avoid copying JS files manually.
- Linting changes
- Enhance Arcanist ESLintLinter to run before/after commands, and set up to use eslint_d
- Update eslint config, and include .eslintignore to avoid linting generated files.
- Include a bunch of eslint-prompted and eslint-generated fixes
- Add no-unused-expression rule to eslint, and fix a few warnings about it
- Other items:
- Refactor cssInput to avoid circular dependency
- Remove a bit of unused code, libraries, dependencies
Test Plan: No behavior changes, all existing tests pass. There are 30 tests fewer reported because `test_gpath.py` was removed (it's been unused for years)
Reviewers: paulfitz
Reviewed By: paulfitz
Subscribers: paulfitz
Differential Revision: https://phab.getgrist.com/D3498
2022-06-27 20:09:41 +00:00
|
|
|
export type { BulkColValues, CellValue, RowRecord };
|
2020-08-13 18:13:16 +00:00
|
|
|
|
2020-09-29 18:53:42 +00:00
|
|
|
// Part of a special CellValue used for comparisons, embedding several versions of a CellValue.
|
2020-11-18 15:54:23 +00:00
|
|
|
export interface AllCellVersions {
|
|
|
|
parent: CellValue;
|
|
|
|
remote: CellValue;
|
|
|
|
local: CellValue;
|
|
|
|
}
|
|
|
|
export type CellVersions = Partial<AllCellVersions>;
|
2020-09-29 18:53:42 +00:00
|
|
|
|
2020-07-21 13:20:51 +00:00
|
|
|
export type AddRecord = ['AddRecord', string, number, ColValues];
|
|
|
|
export type BulkAddRecord = ['BulkAddRecord', string, number[], BulkColValues];
|
|
|
|
export type RemoveRecord = ['RemoveRecord', string, number];
|
|
|
|
export type BulkRemoveRecord = ['BulkRemoveRecord', string, number[]];
|
|
|
|
export type UpdateRecord = ['UpdateRecord', string, number, ColValues];
|
|
|
|
export type BulkUpdateRecord = ['BulkUpdateRecord', string, number[], BulkColValues];
|
|
|
|
|
|
|
|
export type ReplaceTableData = ['ReplaceTableData', string, number[], BulkColValues];
|
|
|
|
|
|
|
|
// This is the format in which data comes when we fetch a table from the sandbox.
|
|
|
|
export type TableDataAction = ['TableData', string, number[], BulkColValues];
|
|
|
|
|
|
|
|
export type AddColumn = ['AddColumn', string, string, ColInfo];
|
|
|
|
export type RemoveColumn = ['RemoveColumn', string, string];
|
|
|
|
export type RenameColumn = ['RenameColumn', string, string, string];
|
2022-12-19 16:52:44 +00:00
|
|
|
export type ModifyColumn = ['ModifyColumn', string, string, Partial<ColInfo>];
|
2020-07-21 13:20:51 +00:00
|
|
|
|
|
|
|
export type AddTable = ['AddTable', string, ColInfoWithId[]];
|
|
|
|
export type RemoveTable = ['RemoveTable', string];
|
|
|
|
export type RenameTable = ['RenameTable', string, string];
|
|
|
|
|
|
|
|
export type DocAction = (
|
|
|
|
AddRecord |
|
|
|
|
BulkAddRecord |
|
|
|
|
RemoveRecord |
|
|
|
|
BulkRemoveRecord |
|
|
|
|
UpdateRecord |
|
|
|
|
BulkUpdateRecord |
|
|
|
|
ReplaceTableData |
|
|
|
|
TableDataAction |
|
|
|
|
AddColumn |
|
|
|
|
RemoveColumn |
|
|
|
|
RenameColumn |
|
|
|
|
ModifyColumn |
|
|
|
|
AddTable |
|
|
|
|
RemoveTable |
|
|
|
|
RenameTable
|
|
|
|
);
|
|
|
|
|
|
|
|
// type guards for convenience - see:
|
|
|
|
// https://www.typescriptlang.org/docs/handbook/advanced-types.html#user-defined-type-guards
|
|
|
|
export function isAddRecord(act: DocAction): act is AddRecord { return act[0] === 'AddRecord'; }
|
|
|
|
export function isBulkAddRecord(act: DocAction): act is BulkAddRecord { return act[0] === 'BulkAddRecord'; }
|
|
|
|
export function isRemoveRecord(act: DocAction): act is RemoveRecord { return act[0] === 'RemoveRecord'; }
|
|
|
|
export function isBulkRemoveRecord(act: DocAction): act is BulkRemoveRecord { return act[0] === 'BulkRemoveRecord'; }
|
|
|
|
export function isUpdateRecord(act: DocAction): act is UpdateRecord { return act[0] === 'UpdateRecord'; }
|
|
|
|
export function isBulkUpdateRecord(act: DocAction): act is BulkUpdateRecord { return act[0] === 'BulkUpdateRecord'; }
|
|
|
|
|
|
|
|
export function isReplaceTableData(act: DocAction): act is ReplaceTableData { return act[0] === 'ReplaceTableData'; }
|
|
|
|
|
|
|
|
export function isAddColumn(act: DocAction): act is AddColumn { return act[0] === 'AddColumn'; }
|
|
|
|
export function isRemoveColumn(act: DocAction): act is RemoveColumn { return act[0] === 'RemoveColumn'; }
|
|
|
|
export function isRenameColumn(act: DocAction): act is RenameColumn { return act[0] === 'RenameColumn'; }
|
|
|
|
export function isModifyColumn(act: DocAction): act is ModifyColumn { return act[0] === 'ModifyColumn'; }
|
|
|
|
|
|
|
|
export function isAddTable(act: DocAction): act is AddTable { return act[0] === 'AddTable'; }
|
|
|
|
export function isRemoveTable(act: DocAction): act is RemoveTable { return act[0] === 'RemoveTable'; }
|
|
|
|
export function isRenameTable(act: DocAction): act is RenameTable { return act[0] === 'RenameTable'; }
|
|
|
|
|
|
|
|
|
|
|
|
const SCHEMA_ACTIONS = new Set(['AddTable', 'RemoveTable', 'RenameTable', 'AddColumn',
|
|
|
|
'RemoveColumn', 'RenameColumn', 'ModifyColumn']);
|
|
|
|
|
2021-10-25 13:29:06 +00:00
|
|
|
const DATA_ACTIONS = new Set(['AddRecord', 'RemoveRecord', 'UpdateRecord', 'BulkAddRecord',
|
|
|
|
'BulkRemoveRecord', 'BulkUpdateRecord', 'ReplaceTableData', 'TableData']);
|
|
|
|
|
2020-07-21 13:20:51 +00:00
|
|
|
/**
|
|
|
|
* Determines whether a given action is a schema action or not.
|
|
|
|
*/
|
2021-04-26 21:54:09 +00:00
|
|
|
export function isSchemaAction(action: DocAction):
|
|
|
|
action is AddTable | RemoveTable | RenameTable | AddColumn | RemoveColumn | RenameColumn | ModifyColumn {
|
2020-07-21 13:20:51 +00:00
|
|
|
return SCHEMA_ACTIONS.has(action[0]);
|
|
|
|
}
|
|
|
|
|
2023-06-15 20:12:19 +00:00
|
|
|
export function isDataAction(action: DocAction|UserAction):
|
2021-10-25 13:29:06 +00:00
|
|
|
action is AddRecord | RemoveRecord | UpdateRecord |
|
|
|
|
BulkAddRecord | BulkRemoveRecord | BulkUpdateRecord |
|
|
|
|
ReplaceTableData | TableDataAction {
|
2023-06-15 20:12:19 +00:00
|
|
|
return DATA_ACTIONS.has(String(action[0]));
|
2021-10-25 13:29:06 +00:00
|
|
|
}
|
|
|
|
|
2020-07-21 13:20:51 +00:00
|
|
|
/**
|
|
|
|
* Returns the tableId from the action.
|
|
|
|
*/
|
|
|
|
export function getTableId(action: DocAction): string {
|
|
|
|
return action[1]; // It happens to always be in the same position in the action tuple.
|
|
|
|
}
|
|
|
|
|
(core) For exporting XLSX, do it memory-efficiently in a worker thread.
Summary:
- Excel exports were awfully memory-inefficient, causing occasional docWorker
crashes. The fix is to use the "streaming writer" option of ExcelJS
https://github.com/exceljs/exceljs#streaming-xlsx-writercontents. (Empirically
on one example, max memory went down from 3G to 100M)
- It's also CPU intensive and synchronous, and can block node for tens of
seconds. The fix is to use a worker-thread. This diff uses "piscina" library
for a pool of threads.
- Additionally, adds ProcessMonitor that logs memory and cpu usage,
particularly when those change significantly.
- Also introduces request cancellation, so that a long download cancelled by
the user will cancel the work being done in the worker thread.
Test Plan:
Updated previous export tests; memory and CPU performance tested
manually by watching output of ProcessMonitor.
Difference visible in these log excerpts:
Before (total time to serve request 22 sec):
```
Telemetry processMonitor heapUsedMB=2187, heapTotalMB=2234, cpuAverage=1.13, intervalMs=17911
Telemetry processMonitor heapUsedMB=2188, heapTotalMB=2234, cpuAverage=0.66, intervalMs=5005
Telemetry processMonitor heapUsedMB=2188, heapTotalMB=2234, cpuAverage=0, intervalMs=5005
Telemetry processMonitor heapUsedMB=71, heapTotalMB=75, cpuAverage=0.13, intervalMs=5002
```
After (total time to server request 18 sec):
```
Telemetry processMonitor heapUsedMB=109, heapTotalMB=144, cpuAverage=0.5, intervalMs=5001
Telemetry processMonitor heapUsedMB=109, heapTotalMB=144, cpuAverage=1.39, intervalMs=5002
Telemetry processMonitor heapUsedMB=94, heapTotalMB=131, cpuAverage=1.13, intervalMs=5000
Telemetry processMonitor heapUsedMB=94, heapTotalMB=131, cpuAverage=1.35, intervalMs=5001
```
Note in "Before" that heapTotalMB goes up to 2GB in the first case, and "intervalMs" of 17 seconds indicates that node was unresponsive for that long. In the second case, heapTotalMB stays low, and the main thread remains responsive the whole time.
Reviewers: jarek
Reviewed By: jarek
Differential Revision: https://phab.getgrist.com/D3906
2023-06-01 13:09:50 +00:00
|
|
|
export interface TableDataActionSet {
|
|
|
|
[tableId: string]: TableDataAction;
|
|
|
|
}
|
|
|
|
|
2020-07-21 13:20:51 +00:00
|
|
|
// Helper types used in the definitions above.
|
|
|
|
|
|
|
|
export interface ColValues { [colId: string]: CellValue; }
|
|
|
|
export interface ColInfoMap { [colId: string]: ColInfo; }
|
|
|
|
|
|
|
|
export interface ColInfo {
|
|
|
|
type: string;
|
|
|
|
isFormula: boolean;
|
|
|
|
formula: string;
|
|
|
|
}
|
|
|
|
|
|
|
|
export interface ColInfoWithId extends ColInfo {
|
|
|
|
id: string;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Multiple records in column-oriented format, i.e. same as BulkColValues but with a mandatory
|
|
|
|
// 'id' column. This is preferred over TableDataAction in external APIs.
|
|
|
|
export interface TableColValues {
|
|
|
|
id: number[];
|
|
|
|
[colId: string]: CellValue[];
|
|
|
|
}
|
|
|
|
|
2021-08-12 14:48:24 +00:00
|
|
|
// Multiple records in record-oriented format
|
|
|
|
export interface TableRecordValues {
|
|
|
|
records: TableRecordValue[];
|
|
|
|
}
|
|
|
|
|
|
|
|
export interface TableRecordValue {
|
2021-08-17 20:38:55 +00:00
|
|
|
id: number | string;
|
2021-08-12 14:48:24 +00:00
|
|
|
fields: {
|
|
|
|
[colId: string]: CellValue
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2020-07-21 13:20:51 +00:00
|
|
|
// Both UserActions and DocActions are represented as [ActionName, ...actionArgs].
|
|
|
|
// TODO I think it's better to represent DocAction as a Buffer containing the marshalled action.
|
|
|
|
|
|
|
|
export type UserAction = Array<string|number|object|boolean|null|undefined>;
|
|
|
|
|
2022-06-17 18:49:18 +00:00
|
|
|
// Actions that trigger formula calculations in the data engine
|
|
|
|
export const CALCULATING_USER_ACTIONS = new Set(['Calculate', 'UpdateCurrentTime', 'RespondToRequests']);
|
|
|
|
|
2021-10-25 13:29:06 +00:00
|
|
|
export function getNumRows(action: DocAction): number {
|
|
|
|
return !isDataAction(action) ? 0
|
|
|
|
: Array.isArray(action[2]) ? action[2].length
|
|
|
|
: 1;
|
|
|
|
}
|
|
|
|
|
2020-07-21 13:20:51 +00:00
|
|
|
// Convert from TableColValues (used by DocStorage and external APIs) to TableDataAction (used
|
|
|
|
// mainly by the sandbox).
|
|
|
|
export function toTableDataAction(tableId: string, colValues: TableColValues): TableDataAction {
|
|
|
|
const colData = {...colValues}; // Make a copy to avoid changing passed-in arguments.
|
|
|
|
const rowIds: number[] = colData.id;
|
(core) Speed up and upgrade build.
Summary:
- Upgrades to build-related packages:
- Upgrade typescript, related libraries and typings.
- Upgrade webpack, eslint; add tsc-watch, node-dev, eslint_d.
- Build organization changes:
- Build webpack from original typescript, transpiling only; with errors still
reported by a background tsc watching process.
- Typescript-related changes:
- Reduce imports of AWS dependencies (very noticeable speedup)
- Avoid auto-loading global @types
- Client code is now built with isolatedModules flag (for safe transpilation)
- Use allowJs to avoid copying JS files manually.
- Linting changes
- Enhance Arcanist ESLintLinter to run before/after commands, and set up to use eslint_d
- Update eslint config, and include .eslintignore to avoid linting generated files.
- Include a bunch of eslint-prompted and eslint-generated fixes
- Add no-unused-expression rule to eslint, and fix a few warnings about it
- Other items:
- Refactor cssInput to avoid circular dependency
- Remove a bit of unused code, libraries, dependencies
Test Plan: No behavior changes, all existing tests pass. There are 30 tests fewer reported because `test_gpath.py` was removed (it's been unused for years)
Reviewers: paulfitz
Reviewed By: paulfitz
Subscribers: paulfitz
Differential Revision: https://phab.getgrist.com/D3498
2022-06-27 20:09:41 +00:00
|
|
|
delete (colData as BulkColValues).id;
|
2020-07-21 13:20:51 +00:00
|
|
|
return ['TableData', tableId, rowIds, colData];
|
|
|
|
}
|
|
|
|
|
|
|
|
// Convert from TableDataAction (used mainly by the sandbox) to TableColValues (used by DocStorage
|
|
|
|
// and external APIs).
|
2022-12-21 16:40:00 +00:00
|
|
|
// Also accepts a TableDataAction nested as a tableData member of a larger structure,
|
|
|
|
// for convenience in dealing with the result of fetches.
|
|
|
|
export function fromTableDataAction(tableData: TableDataAction|{tableData: TableDataAction}): TableColValues {
|
|
|
|
const data = ('tableData' in tableData) ? tableData.tableData : tableData;
|
|
|
|
const rowIds: number[] = data[2];
|
|
|
|
const colValues: BulkColValues = data[3];
|
2020-07-21 13:20:51 +00:00
|
|
|
return {id: rowIds, ...colValues};
|
|
|
|
}
|
2022-01-04 11:54:06 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Convert a list of rows into an object with columns of values, used for
|
|
|
|
* BulkAddRecord/BulkUpdateRecord actions.
|
|
|
|
*/
|
2022-07-21 14:46:26 +00:00
|
|
|
export function getColValues(records: Partial<RowRecord>[]): BulkColValues {
|
2022-01-04 11:54:06 +00:00
|
|
|
const colIdSet = new Set<string>();
|
|
|
|
for (const r of records) {
|
|
|
|
for (const c of Object.keys(r)) {
|
|
|
|
if (c !== 'id') {
|
|
|
|
colIdSet.add(c);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
const result: BulkColValues = {};
|
|
|
|
for (const colId of colIdSet) {
|
2022-07-21 14:46:26 +00:00
|
|
|
result[colId] = records.map(r => r[colId]!);
|
2022-01-04 11:54:06 +00:00
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
2022-12-21 16:40:00 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Extract the col ids mentioned in a record-related DocAction as a list
|
|
|
|
* (even if the action is not a bulk action). Returns undefined if no col ids
|
|
|
|
* mentioned.
|
|
|
|
*/
|
|
|
|
export function getColIdsFromDocAction(docActions: RemoveRecord | BulkRemoveRecord | AddRecord |
|
|
|
|
BulkAddRecord | UpdateRecord | BulkUpdateRecord | ReplaceTableData |
|
|
|
|
TableDataAction): string[] | undefined {
|
|
|
|
if (docActions[3]) { return Object.keys(docActions[3]); }
|
|
|
|
return undefined;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Extract column values for a particular column as CellValue[] from a
|
|
|
|
* record-related DocAction. Undefined if absent.
|
|
|
|
*/
|
|
|
|
export function getColValuesFromDocAction(docAction: RemoveRecord | BulkRemoveRecord | AddRecord |
|
|
|
|
BulkAddRecord | UpdateRecord | BulkUpdateRecord | ReplaceTableData |
|
|
|
|
TableDataAction, colId: string): CellValue[]|undefined {
|
|
|
|
const colValues = docAction[3];
|
|
|
|
if (!colValues) { return undefined; }
|
|
|
|
const cellValues = colValues[colId];
|
|
|
|
if (!cellValues) { return undefined; }
|
|
|
|
if (Array.isArray(docAction[2])) {
|
|
|
|
return cellValues as CellValue[];
|
|
|
|
} else {
|
|
|
|
return [cellValues as CellValue];
|
|
|
|
}
|
|
|
|
}
|