From faec8177abd50429f7292d2cb52bd4de8916ceed Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Tue, 7 Dec 2021 13:21:16 +0200 Subject: [PATCH] (core) Use MetaTableData more Summary: Add more method overrides to MetaTableData for extra type safety. Use MetaTableData, MetaRowRecord, and getMetaTable in more places. Test Plan: Mostly it just has to compile. Tested manually that types are being checked more strictly now, e.g. by adding a typo to property names. Some type casting has also been removed. Reviewers: dsagal Reviewed By: dsagal Subscribers: dsagal Differential Revision: https://phab.getgrist.com/D3168 --- app/client/aclui/AccessRules.ts | 7 ++-- app/client/components/GristDoc.ts | 2 +- app/client/components/duplicatePage.ts | 4 +-- app/client/models/DocData.ts | 8 ++++- app/client/models/TableData.ts | 21 ++++++----- app/client/widgets/AttachmentsEditor.ts | 12 +++---- app/client/widgets/AttachmentsWidget.ts | 22 ++++++------ app/common/ACLRuleCollection.ts | 32 ++++++++--------- app/common/GranularAccessClause.ts | 9 ++--- app/common/TableData.ts | 45 ++++++++++++++++++----- app/server/lib/ActiveDoc.ts | 45 ++++++++++++----------- app/server/lib/DocPluginManager.ts | 4 +-- app/server/lib/ExpandedQuery.ts | 6 ++-- app/server/lib/Export.ts | 48 +++++++++++-------------- app/server/lib/ExportCSV.ts | 2 +- app/server/lib/GranularAccess.ts | 8 ++--- app/server/lib/OnDemandActions.ts | 4 +-- app/server/lib/Triggers.ts | 4 +-- 18 files changed, 157 insertions(+), 126 deletions(-) diff --git a/app/client/aclui/AccessRules.ts b/app/client/aclui/AccessRules.ts index 8f9209c3..bf8a8a15 100644 --- a/app/client/aclui/AccessRules.ts +++ b/app/client/aclui/AccessRules.ts @@ -38,6 +38,7 @@ import {isHiddenCol} from 'app/common/gristTypes'; import {isObject} from 'app/common/gutil'; import * as roles from 'app/common/roles'; import {SchemaTypes} from 'app/common/schema'; +import {MetaRowRecord} from 'app/common/TableData'; import {ANONYMOUS_USER_EMAIL, EVERYONE_EMAIL, getRealAccess} from 'app/common/UserAPI'; import { BaseObservable, @@ -216,11 +217,11 @@ export class AccessRules extends Disposable { // ACL tables (they may have changed by other users). So our changes will win. const docData = this._gristDoc.docData; - const resourcesTable = docData.getTable('_grist_ACLResources')!; - const rulesTable = docData.getTable('_grist_ACLRules')!; + const resourcesTable = docData.getMetaTable('_grist_ACLResources'); + const rulesTable = docData.getMetaTable('_grist_ACLRules'); // Add/remove resources to have just the ones we need. - const newResources: RowRecord[] = flatten( + const newResources: MetaRowRecord<'_grist_ACLResources'>[] = flatten( [{tableId: '*', colIds: '*'}], this._specialRules.get()?.getResources() || [], ...this._tableRules.get().map(t => t.getResources())) diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index 6b5797b0..1c73d25c 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -675,7 +675,7 @@ export class GristDoc extends DisposableWithEvents { } public hasGranularAccessRules(): boolean { - const rulesTable = this.docData.getTable('_grist_ACLRules')!; + const rulesTable = this.docData.getMetaTable('_grist_ACLRules'); // To check if there are rules, ignore the default no-op rule created for an older incarnation // of ACLs. It exists in older documents, and is still created for new ones. We detect it by // the use of the deprecated 'permissions' field, and not the new 'permissionsText' field. diff --git a/app/client/components/duplicatePage.ts b/app/client/components/duplicatePage.ts index 64db9561..6395b70f 100644 --- a/app/client/components/duplicatePage.ts +++ b/app/client/components/duplicatePage.ts @@ -90,7 +90,7 @@ async function updateViewSections(gristDoc: GristDoc, destViewSections: ViewSect const records: RowRecord[] = []; for (const srcViewSection of srcViewSections) { const viewSectionLayoutSpec = patchLayoutSpec(srcViewSection.layoutSpecObj.peek(), fieldsMap); - const record = gristDoc.docData.getTable('_grist_Views_section')!.getRecord(srcViewSection.getRowId())!; + const record = gristDoc.docData.getMetaTable('_grist_Views_section').getRecord(srcViewSection.getRowId())!; records.push({ ...record, layoutSpec: JSON.stringify(viewSectionLayoutSpec), @@ -126,7 +126,7 @@ async function updateViewFields(gristDoc: GristDoc, destViewSections: ViewSectio const srcViewFields: ViewFieldRec[] = srcViewSection!.viewFields.peek().peek(); const parentId = destViewSection!.getRowId(); for (const field of srcViewFields) { - const record = docData.getTable('_grist_Views_section_field')!.getRecord(field.getRowId())!; + const record = docData.getMetaTable('_grist_Views_section_field').getRecord(field.getRowId())!; fieldsToAdd.push({...record, parentId}); } } diff --git a/app/client/models/DocData.ts b/app/client/models/DocData.ts index 68d24280..e7ca092c 100644 --- a/app/client/models/DocData.ts +++ b/app/client/models/DocData.ts @@ -5,10 +5,11 @@ */ import {DocComm} from 'app/client/components/DocComm'; -import {TableData} from 'app/client/models/TableData'; +import {MetaTableData, TableData} from 'app/client/models/TableData'; import {ApplyUAOptions, ApplyUAResult} from 'app/common/ActiveDocAPI'; import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions'; import {DocData as BaseDocData} from 'app/common/DocData'; +import {SchemaTypes} from 'app/common/schema'; import {ColTypeMap} from 'app/common/TableData'; import * as bluebird from 'bluebird'; import {Emitter} from 'grainjs'; @@ -51,6 +52,11 @@ export class DocData extends BaseDocData { return super.getTable(tableId) as TableData; } + // Version of inherited getMetaTable() which returns the enhanced TableData type. + public getMetaTable(tableId: TableId): MetaTableData { + return super.getMetaTable(tableId) as any; + } + /** * Finds up to n most likely target columns for the given values in the document. */ diff --git a/app/client/models/TableData.ts b/app/client/models/TableData.ts index d0fd0ea2..62e58d46 100644 --- a/app/client/models/TableData.ts +++ b/app/client/models/TableData.ts @@ -1,19 +1,20 @@ /** * TableData maintains a single table's data. */ -import { ColumnACIndexes } from 'app/client/models/ColumnACIndexes'; -import { ColumnCache } from 'app/client/models/ColumnCache'; -import { DocData } from 'app/client/models/DocData'; -import { DocAction, ReplaceTableData, TableDataAction, UserAction } from 'app/common/DocActions'; -import { isRaisedException } from 'app/common/gristTypes'; -import { countIf } from 'app/common/gutil'; -import { TableData as BaseTableData, ColTypeMap } from 'app/common/TableData'; -import { Emitter } from 'grainjs'; +import {ColumnACIndexes} from 'app/client/models/ColumnACIndexes'; +import {ColumnCache} from 'app/client/models/ColumnCache'; +import {DocData} from 'app/client/models/DocData'; +import {DocAction, ReplaceTableData, TableDataAction, UserAction} from 'app/common/DocActions'; +import {isRaisedException} from 'app/common/gristTypes'; +import {countIf} from 'app/common/gutil'; +import {SchemaTypes} from 'app/common/schema'; +import {ColTypeMap, MetaTableData as MetaTableDataBase, TableData as TableDataBase} from 'app/common/TableData'; +import {Emitter} from 'grainjs'; /** * TableData class to maintain a single table's data. */ -export class TableData extends BaseTableData { +export class TableData extends TableDataBase { public readonly tableActionEmitter = new Emitter(); public readonly dataLoadedEmitter = new Emitter(); @@ -108,3 +109,5 @@ export class TableData extends BaseTableData { return applied; } } + +export type MetaTableData = MetaTableDataBase & TableData; diff --git a/app/client/widgets/AttachmentsEditor.ts b/app/client/widgets/AttachmentsEditor.ts index 42873ef4..fefb6631 100644 --- a/app/client/widgets/AttachmentsEditor.ts +++ b/app/client/widgets/AttachmentsEditor.ts @@ -7,7 +7,7 @@ import {dom, LiveIndex, makeLiveIndex, styled} from 'grainjs'; import {DocComm} from 'app/client/components/DocComm'; import {selectFiles, uploadFiles} from 'app/client/lib/uploads'; import {DocData} from 'app/client/models/DocData'; -import {TableData} from 'app/client/models/TableData'; +import {MetaTableData} from 'app/client/models/TableData'; import {basicButton, basicButtonLink, cssButtonGroup} from 'app/client/ui2018/buttons'; import {colors, testId, vars} from 'app/client/ui2018/cssVars'; import {editableLabel} from 'app/client/ui2018/editableLabel'; @@ -47,7 +47,7 @@ interface Attachment { * download, add or remove attachments in the edited cell. */ export class AttachmentsEditor extends NewBaseEditor { - private _attachmentsTable: TableData; + private _attachmentsTable: MetaTableData<'_grist_Attachments'>; private _docComm: DocComm; private _rowIds: MutableObsArray; @@ -64,15 +64,15 @@ export class AttachmentsEditor extends NewBaseEditor { // editValue is abused slightly to indicate a 1-based index of the attachment. const initRowIndex: number|undefined = (options.editValue && parseInt(options.editValue, 0) - 1) || 0; - this._attachmentsTable = docData.getTable('_grist_Attachments')!; + this._attachmentsTable = docData.getMetaTable('_grist_Attachments'); this._docComm = docData.docComm; this._rowIds = obsArray(Array.isArray(cellValue) ? cellValue.slice(1) as number[] : []); this._attachments = computedArray(this._rowIds, (val: number): Attachment => { - const fileIdent: string = this._attachmentsTable.getValue(val, 'fileIdent') as string; + const fileIdent: string = this._attachmentsTable.getValue(val, 'fileIdent')!; const fileType = mimeTypes.lookup(fileIdent) || 'application/octet-stream'; const filename: Observable = - observable(this._attachmentsTable.getValue(val, 'fileName') as string); + observable(this._attachmentsTable.getValue(val, 'fileName')!); return { rowId: val, fileIdent, @@ -187,7 +187,7 @@ export class AttachmentsEditor extends NewBaseEditor { private async _renameAttachment(att: Attachment, fileName: string): Promise { await this._attachmentsTable.sendTableAction(['UpdateRecord', att.rowId, {fileName}]); // Update the observable, since it's not on its own observing changes. - att.filename.set(this._attachmentsTable.getValue(att.rowId, 'fileName') as string); + att.filename.set(this._attachmentsTable.getValue(att.rowId, 'fileName')!); } private _getUrl(fileIdent: string, filename: string, inline?: boolean): string { diff --git a/app/client/widgets/AttachmentsWidget.ts b/app/client/widgets/AttachmentsWidget.ts index 5feab2b8..476b671b 100644 --- a/app/client/widgets/AttachmentsWidget.ts +++ b/app/client/widgets/AttachmentsWidget.ts @@ -7,7 +7,7 @@ import {cssRow} from 'app/client/ui/RightPanel'; import {colors, vars} from 'app/client/ui2018/cssVars'; import {NewAbstractWidget} from 'app/client/widgets/NewAbstractWidget'; import {encodeQueryParams} from 'app/common/gutil'; -import {TableData} from 'app/common/TableData'; +import {MetaTableData} from 'app/client/models/TableData'; import {UploadResult} from 'app/common/uploads'; import {extname} from 'path'; @@ -66,7 +66,7 @@ export interface SavingObservable extends ko.Observable { */ export class AttachmentsWidget extends NewAbstractWidget { - private _attachmentsTable: TableData; + private _attachmentsTable: MetaTableData<'_grist_Attachments'>; private _height: SavingObservable; constructor(field: any) { @@ -74,7 +74,7 @@ export class AttachmentsWidget extends NewAbstractWidget { // TODO: the Attachments table currently treated as metadata, and loaded on open, // but should probably be loaded on demand as it contains user data, which may be large. - this._attachmentsTable = this._getDocData().getTable('_grist_Attachments')!; + this._attachmentsTable = this._getDocData().getMetaTable('_grist_Attachments'); this._height = this.options.prop('height') as SavingObservable; @@ -122,12 +122,12 @@ export class AttachmentsWidget extends NewAbstractWidget { } protected _buildAttachment(value: number, allValues: Computed): Element { - const filename: string = this._attachmentsTable.getValue(value, 'fileName') as string; - const fileIdent: string = this._attachmentsTable.getValue(value, 'fileIdent') as string; - const height: number = this._attachmentsTable.getValue(value, 'imageHeight') as number; - const width: number = this._attachmentsTable.getValue(value, 'imageWidth') as number; - const hasPreview: boolean = Boolean(height); - const ratio: number = hasPreview ? (width / height) : 1; + const filename = this._attachmentsTable.getValue(value, 'fileName')!; + const fileIdent = this._attachmentsTable.getValue(value, 'fileIdent')!; + const height = this._attachmentsTable.getValue(value, 'imageHeight')!; + const width = this._attachmentsTable.getValue(value, 'imageWidth')!; + const hasPreview = Boolean(height); + const ratio = hasPreview ? (width / height) : 1; return attachmentPreview({title: filename}, // Add a filename tooltip to the previews. dom.style('height', (use) => `${use(this._height)}px`), @@ -146,7 +146,7 @@ export class AttachmentsWidget extends NewAbstractWidget { // Returns the attachment download url. private _getUrl(rowId: number): string { - const ident = this._attachmentsTable.getValue(rowId, 'fileIdent') as string; + const ident = this._attachmentsTable.getValue(rowId, 'fileIdent'); if (!ident) { return ''; } else { @@ -154,7 +154,7 @@ export class AttachmentsWidget extends NewAbstractWidget { return docComm.docUrl('attachment') + '?' + encodeQueryParams({ ...docComm.getUrlParams(), ident, - name: this._attachmentsTable.getValue(rowId, 'fileName') as string + name: this._attachmentsTable.getValue(rowId, 'fileName') }); } } diff --git a/app/common/ACLRuleCollection.ts b/app/common/ACLRuleCollection.ts index 67e8502f..bb418146 100644 --- a/app/common/ACLRuleCollection.ts +++ b/app/common/ACLRuleCollection.ts @@ -1,9 +1,9 @@ -import { parsePermissions } from 'app/common/ACLPermissions'; -import { ILogger } from 'app/common/BaseAPI'; -import { CellValue, RowRecord } from 'app/common/DocActions'; -import { DocData } from 'app/common/DocData'; -import { AclMatchFunc, ParsedAclFormula, RulePart, RuleSet, UserAttributeRule } from 'app/common/GranularAccessClause'; -import { getSetMapValue } from 'app/common/gutil'; +import {parsePermissions} from 'app/common/ACLPermissions'; +import {ILogger} from 'app/common/BaseAPI'; +import {DocData} from 'app/common/DocData'; +import {AclMatchFunc, ParsedAclFormula, RulePart, RuleSet, UserAttributeRule} from 'app/common/GranularAccessClause'; +import {getSetMapValue} from 'app/common/gutil'; +import {MetaRowRecord} from 'app/common/TableData'; import sortBy = require('lodash/sortBy'); const defaultMatchFunc: AclMatchFunc = () => true; @@ -231,8 +231,8 @@ export class ACLRuleCollection { * Check that all references to table and column IDs in ACL rules are valid. */ public checkDocEntities(docData: DocData) { - const tablesTable = docData.getTable('_grist_Tables')!; - const columnsTable = docData.getTable('_grist_Tables_column')!; + const tablesTable = docData.getMetaTable('_grist_Tables'); + const columnsTable = docData.getMetaTable('_grist_Tables_column'); // Collect valid tableIds and check rules against those. const validTableIds = new Set(tablesTable.getColValues('tableId')); @@ -242,9 +242,9 @@ export class ACLRuleCollection { } // Collect valid columns, grouped by tableRef (rowId of table record). - const validColumns = new Map>(); // Map from tableRef to set of colIds. - const colTableRefs = columnsTable.getColValues('parentId')!; - for (const [i, colId] of columnsTable.getColValues('colId')!.entries()) { + const validColumns = new Map>(); // Map from tableRef to set of colIds. + const colTableRefs = columnsTable.getColValues('parentId'); + for (const [i, colId] of columnsTable.getColValues('colId').entries()) { getSetMapValue(validColumns, colTableRefs[i], () => new Set()).add(colId); } @@ -302,14 +302,14 @@ export interface ReadAclResults { * UserAttributeRules. This is used by both client-side code and server-side. */ function readAclRules(docData: DocData, {log, compile}: ReadAclOptions): ReadAclResults { - const resourcesTable = docData.getTable('_grist_ACLResources')!; - const rulesTable = docData.getTable('_grist_ACLRules')!; + const resourcesTable = docData.getMetaTable('_grist_ACLResources'); + const rulesTable = docData.getMetaTable('_grist_ACLRules'); const ruleSets: RuleSet[] = []; const userAttributes: UserAttributeRule[] = []; // Group rules by resource first, ordering by rulePos. Each group will become a RuleSet. - const rulesByResource = new Map(); + const rulesByResource = new Map>>(); for (const ruleRecord of sortBy(rulesTable.getRecords(), 'rulePos')) { getSetMapValue(rulesByResource, ruleRecord.resource, () => []).push(ruleRecord); } @@ -325,8 +325,8 @@ function readAclRules(docData: DocData, {log, compile}: ReadAclOptions): ReadAcl // intentionally ignore and skip. continue; } - const tableId = resourceRec.tableId as string; - const colIds = resourceRec.colIds === '*' ? '*' : (resourceRec.colIds as string).split(','); + const tableId = resourceRec.tableId; + const colIds = resourceRec.colIds === '*' ? '*' : resourceRec.colIds.split(','); const body: RulePart[] = []; for (const rule of rules) { diff --git a/app/common/GranularAccessClause.ts b/app/common/GranularAccessClause.ts index 511ab4fb..b3e21657 100644 --- a/app/common/GranularAccessClause.ts +++ b/app/common/GranularAccessClause.ts @@ -1,6 +1,7 @@ -import { PartialPermissionSet } from 'app/common/ACLPermissions'; -import { CellValue, RowRecord } from 'app/common/DocActions'; -import { Role } from './roles'; +import {PartialPermissionSet} from 'app/common/ACLPermissions'; +import {CellValue, RowRecord} from 'app/common/DocActions'; +import {MetaRowRecord} from 'app/common/TableData'; +import {Role} from './roles'; export interface RuleSet { tableId: '*' | string; @@ -11,7 +12,7 @@ export interface RuleSet { } export interface RulePart { - origRecord?: RowRecord; // Original record used to create this RulePart. + origRecord?: MetaRowRecord<'_grist_ACLRules'>; // Original record used to create this RulePart. aclFormula: string; permissions: PartialPermissionSet; permissionsText: string; // The text version of PermissionSet, as stored. diff --git a/app/common/TableData.ts b/app/common/TableData.ts index d9dcd86c..360c6817 100644 --- a/app/common/TableData.ts +++ b/app/common/TableData.ts @@ -13,7 +13,6 @@ import fromPairs = require('lodash/fromPairs'); export interface ColTypeMap { [colId: string]: string; } -type RowFunc = (rowId: number) => T; type UIRowFunc = (rowId: UIRowId) => T; interface ColData { @@ -325,7 +324,7 @@ export class TableData extends ActionDispatcher implements SkippableRows { * Returns the first rowId matching the given filters, or 0 if no match. If there are multiple * matches, it is unspecified which will be returned. */ - public findMatchingRowId(properties: {[key: string]: CellValue}): number { + public findMatchingRowId(properties: {[key: string]: CellValue | undefined}): number { const props = Object.keys(properties).map(p => ({col: this._columns.get(p)!, value: properties[p]})); if (!props.every((p) => p.col)) { return 0; @@ -480,7 +479,15 @@ export class TableData extends ActionDispatcher implements SkippableRows { } } -export type MetaRowRecord = SchemaTypes[TableId] & RowRecord; +// A type safe record of a meta table with types as defined in schema.ts +// '&' is used because declaring the id field and the index signature in one block gives a syntax error. +// The second part is basically equivalent to SchemaTypes[TableId] +// but TS sees that as incompatible with RowRecord and doesn't allow simple overrides in MetaTableData. +export type MetaRowRecord = + { id: number } & + { [ColId in keyof SchemaTypes[TableId]]: SchemaTypes[TableId][ColId] & CellValue }; + +type MetaColId = keyof MetaRowRecord & string; /** * Behaves the same as TableData, but uses SchemaTypes for type safety of its columns. @@ -490,6 +497,11 @@ export class MetaTableData extends TableData super(tableId, tableData, colTypes); } + public getValue>(rowId: number, colId: ColId): + MetaRowRecord[ColId] | undefined { + return super.getValue(rowId, colId) as any; + } + public getRecords(): Array> { return super.getRecords() as any; } @@ -498,14 +510,31 @@ export class MetaTableData extends TableData return super.getRecord(rowId) as any; } - /** - * Same as getRowPropFunc, but I couldn't get a direct override to compile. - */ - public getMetaRowPropFunc( + public filterRecords(properties: Partial>): Array> { + return super.filterRecords(properties) as any; + } + + public findMatchingRowId(properties: Partial>): number { + return super.findMatchingRowId(properties); + } + + public getRowPropFunc>( colId: ColId - ): RowFunc { + ): UIRowFunc[ColId]> { return super.getRowPropFunc(colId as any) as any; } + + public getColValues>( + colId: ColId + ): ReadonlyArray[ColId]> { + return super.getColValues(colId) as any; + } + + public findRow>( + colId: ColId, colValue: MetaRowRecord[ColId] + ): number { + return super.findRow(colId, colValue); + } } function reassignArray(targetArray: T[], sourceArray: T[]): void { diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 7af3dae5..d6e3b1e5 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -4,20 +4,9 @@ * change events. */ -import { ActionSummary } from "app/common/ActionSummary"; -import { DocTriggers } from "app/server/lib/Triggers"; -import * as assert from 'assert'; -import {Mutex} from 'async-mutex'; -import * as bluebird from 'bluebird'; -import {EventEmitter} from 'events'; -import {IMessage, MsgType} from 'grain-rpc'; -import * as imageSize from 'image-size'; -import * as moment from 'moment-timezone'; -import fetch from 'node-fetch'; -import * as tmp from 'tmp'; - import {getEnvContent, LocalActionBundle, SandboxActionBundle, UserActionBundle} from 'app/common/ActionBundle'; -import { ActionGroup, MinimalActionGroup} from 'app/common/ActionGroup'; +import {ActionGroup, MinimalActionGroup} from 'app/common/ActionGroup'; +import {ActionSummary} from "app/common/ActionSummary"; import { ApplyUAOptions, ApplyUAResult, @@ -36,7 +25,6 @@ import {mapGetOrSet, MapWithTTL} from 'app/common/AsyncCreate'; import { CellValue, DocAction, - RowRecord, TableDataAction, TableRecordValue, toTableDataAction, @@ -50,6 +38,7 @@ import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccess import {byteString, countIf, safeJsonParse} from 'app/common/gutil'; import {InactivityTimer} from 'app/common/InactivityTimer'; import {schema, SCHEMA_VERSION} from 'app/common/schema'; +import {MetaRowRecord} from 'app/common/TableData'; import {FetchUrlOptions, UploadResult} from 'app/common/uploads'; import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI'; import {ParseOptions} from 'app/plugin/FileParserAPI'; @@ -66,7 +55,17 @@ import * as log from 'app/server/lib/log'; import {LogMethods} from "app/server/lib/LogMethods"; import {shortDesc} from 'app/server/lib/shortDesc'; import {TableMetadataLoader} from 'app/server/lib/TableMetadataLoader'; +import {DocTriggers} from "app/server/lib/Triggers"; import {fetchURL, FileUploadInfo, globalUploadSet, UploadInfo} from 'app/server/lib/uploads'; +import * as assert from 'assert'; +import {Mutex} from 'async-mutex'; +import * as bluebird from 'bluebird'; +import {EventEmitter} from 'events'; +import {IMessage, MsgType} from 'grain-rpc'; +import * as imageSize from 'image-size'; +import * as moment from 'moment-timezone'; +import fetch from 'node-fetch'; +import * as tmp from 'tmp'; import {ActionHistory} from './ActionHistory'; import {ActionHistoryImpl} from './ActionHistoryImpl'; @@ -89,8 +88,8 @@ import {findOrAddAllEnvelope, Sharing} from './Sharing'; import cloneDeep = require('lodash/cloneDeep'); import flatten = require('lodash/flatten'); import remove = require('lodash/remove'); -import zipObject = require('lodash/zipObject'); import without = require('lodash/without'); +import zipObject = require('lodash/zipObject'); bluebird.promisifyAll(tmp); @@ -608,14 +607,14 @@ export class ActiveDoc extends EventEmitter { * Returns the record from _grist_Attachments table for the given attachment ID, * or throws an error if not found. */ - public getAttachmentMetadata(attId: number|string): RowRecord { + public getAttachmentMetadata(attId: number|string): MetaRowRecord<'_grist_Attachments'> { // docData should always be available after loadDoc() or createDoc(). if (!this.docData) { throw new Error("No doc data"); } // Parse strings into numbers to make more convenient to call from route handlers. const attachmentId: number = (typeof attId === 'string') ? parseInt(attId, 10) : attId; - const attRecord = this.docData.getTable('_grist_Attachments')!.getRecord(attachmentId); + const attRecord = this.docData.getMetaTable('_grist_Attachments').getRecord(attachmentId); if (!attRecord) { throw new ApiError(`Attachment not found: ${attId}`, 404); } @@ -1116,14 +1115,14 @@ export class ActiveDoc extends EventEmitter { throw new Error('Cannot list ACL resources'); } const result: {[tableId: string]: string[]} = {}; - const tables = this.docData.getTable('_grist_Tables')!; - for (const tableId of tables.getColValues('tableId')!) { - result[tableId as string] = ['id']; + const tables = this.docData.getMetaTable('_grist_Tables'); + for (const tableId of tables.getColValues('tableId')) { + result[tableId] = ['id']; } - const columns = this.docData.getTable('_grist_Tables_column')!; + const columns = this.docData.getMetaTable('_grist_Tables_column'); for (const col of columns.getRecords()) { - const tableId = tables.getValue(col.parentId as number, 'tableId'); - result[tableId as string].push(col.colId as string); + const tableId = tables.getValue(col.parentId, 'tableId')!; + result[tableId].push(col.colId); } return result; } diff --git a/app/server/lib/DocPluginManager.ts b/app/server/lib/DocPluginManager.ts index 5557bc38..5a8edc7c 100644 --- a/app/server/lib/DocPluginManager.ts +++ b/app/server/lib/DocPluginManager.ts @@ -33,8 +33,8 @@ class GristDocAPIImpl implements GristDocAPI { public async getDocName() { return this._activeDoc.docName; } public async listTables(): Promise { - const table = this._activeDoc.docData!.getTable('_grist_Tables')!; - return (table.getColValues('tableId') as string[]) + const table = this._activeDoc.docData!.getMetaTable('_grist_Tables'); + return table.getColValues('tableId') .filter(id => !id.startsWith("GristSummary_")).sort(); } diff --git a/app/server/lib/ExpandedQuery.ts b/app/server/lib/ExpandedQuery.ts index 0bf668ef..dbd7ee4d 100644 --- a/app/server/lib/ExpandedQuery.ts +++ b/app/server/lib/ExpandedQuery.ts @@ -56,8 +56,8 @@ export function expandQuery(iquery: ServerQuery, docData: DocData, onDemandFormu // Iterate through all formulas, adding joins and selects as we go. if (onDemandFormulas) { // Look up the main table for the query. - const tables = docData.getTable('_grist_Tables')!; - const columns = docData.getTable('_grist_Tables_column')!; + const tables = docData.getMetaTable('_grist_Tables'); + const columns = docData.getMetaTable('_grist_Tables_column'); const tableRef = tables.findRow('tableId', query.tableId); if (!tableRef) { throw new ApiError('table not found', 404); } @@ -81,7 +81,7 @@ export function expandQuery(iquery: ServerQuery, docData: DocData, onDemandFormu let error = ""; if (formula.kind === 'foreignColumn') { const altTableId = references.get(formula.refColId); - const altTableRef = tables.findRow('tableId', altTableId); + const altTableRef = tables.findRow('tableId', altTableId!); if (altTableId && altTableRef) { const altColumn = columns.filterRecords({parentId: altTableRef, isFormula: false, colId: formula.colId}); // TODO: deal with a formula column in the other table. diff --git a/app/server/lib/Export.ts b/app/server/lib/Export.ts index 55c92833..ee35bab7 100644 --- a/app/server/lib/Export.ts +++ b/app/server/lib/Export.ts @@ -1,6 +1,5 @@ import {ApiError} from 'app/common/ApiError'; import {buildColFilter} from 'app/common/ColumnFilterFunc'; -import {RowRecord} from 'app/common/DocActions'; import {DocData} from 'app/common/DocData'; import {DocumentSettings} from 'app/common/DocumentSettings'; import * as gristTypes from 'app/common/gristTypes'; @@ -9,7 +8,7 @@ import {buildRowFilter} from 'app/common/RowFilterFunc'; import {SchemaTypes} from 'app/common/schema'; import {SortFunc} from 'app/common/SortFunc'; import {Sort} from 'app/common/SortSpec'; -import {TableData} from 'app/common/TableData'; +import {MetaRowRecord, MetaTableData} from 'app/common/TableData'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {RequestWithLogin} from 'app/server/lib/Authorizer'; import {docSessionFromRequest} from 'app/server/lib/DocSession'; @@ -103,12 +102,14 @@ function safe(value: T, msg: string) { } // Helper to for getting table from docData. -const safeTable = (docData: DocData, name: keyof SchemaTypes) => safe(docData.getTable(name), - `No table '${name}' in document with id ${docData}`); +function safeTable(docData: DocData, name: TableId) { + return safe(docData.getMetaTable(name), `No table '${name}' in document with id ${docData}`); +} // Helper for getting record safe -const safeRecord = (table: TableData, id: number) => safe(table.getRecord(id), - `No record ${id} in table ${table.tableId}`); +function safeRecord(table: MetaTableData, id: number) { + return safe(table.getRecord(id), `No record ${id} in table ${table.tableId}`); +} /** * Builds export for all raw tables that are in doc. @@ -138,9 +139,9 @@ export async function exportTable( req: express.Request): Promise { const docData = safe(activeDoc.docData, "No docData in active document"); const tables = safeTable(docData, '_grist_Tables'); - const table = safeRecord(tables, tableId) as GristTables; - const tableColumns = (safeTable(docData, '_grist_Tables_column') - .getRecords() as GristTablesColumn[]) + const table = safeRecord(tables, tableId); + const tableColumns = safeTable(docData, '_grist_Tables_column') + .getRecords() // remove manual sort column .filter(col => col.colId !== gristTypes.MANUALSORT); // Produce a column description matching what user will see / expect to export @@ -183,11 +184,11 @@ export async function exportTable( if (table.primaryViewId) { const viewId = table.primaryViewId; const views = safeTable(docData, '_grist_Views'); - const view = safeRecord(views, viewId) as GristView; + const view = safeRecord(views, viewId); tableName = view.name; } - const docInfo = safeRecord(safeTable(docData, '_grist_DocInfo'), 1) as DocInfo; + const docInfo = safeRecord(safeTable(docData, '_grist_DocInfo'), 1); const docSettings = gutil.safeJsonParse(docInfo.documentSettings, {}); return { tableName, @@ -211,15 +212,15 @@ export async function exportSection( const docData = safe(activeDoc.docData, "No docData in active document"); const viewSections = safeTable(docData, '_grist_Views_section'); - const viewSection = safeRecord(viewSections, viewSectionId) as GristViewsSection; + const viewSection = safeRecord(viewSections, viewSectionId); const tables = safeTable(docData, '_grist_Tables'); - const table = safeRecord(tables, viewSection.tableRef) as GristTables; + const table = safeRecord(tables, viewSection.tableRef); const columns = safeTable(docData, '_grist_Tables_column') - .filterRecords({ parentId: table.id }) as GristTablesColumn[]; + .filterRecords({parentId: table.id}); const viewSectionFields = safeTable(docData, '_grist_Views_section_field'); - const fields = viewSectionFields.filterRecords({ parentId: viewSection.id }) as GristViewsSectionField[]; + const fields = viewSectionFields.filterRecords({parentId: viewSection.id}); const savedFilters = safeTable(docData, '_grist_Filters') - .filterRecords({ viewSectionRef: viewSection.id }) as GristFilter[]; + .filterRecords({viewSectionRef: viewSection.id}); const tableColsById = _.indexBy(columns, 'id'); const fieldsByColRef = _.indexBy(fields, 'colRef'); @@ -279,7 +280,7 @@ export async function exportSection( // filter rows numbers rowIds = rowIds.filter(rowFilter); - const docInfo = safeRecord(safeTable(docData, '_grist_DocInfo'), 1) as DocInfo; + const docInfo = safeRecord(safeTable(docData, '_grist_DocInfo'), 1); const docSettings = gutil.safeJsonParse(docInfo.documentSettings, {}); return { @@ -292,17 +293,8 @@ export async function exportSection( }; } -// Type helpers for types used in this export -type RowModel = RowRecord & { - [ColId in keyof SchemaTypes[TName]]: SchemaTypes[TName][ColId]; -}; -type GristViewsSection = RowModel<'_grist_Views_section'> -type GristTables = RowModel<'_grist_Tables'> -type GristViewsSectionField = RowModel<'_grist_Views_section_field'> -type GristTablesColumn = RowModel<'_grist_Tables_column'> -type GristView = RowModel<'_grist_Views'> -type GristFilter = RowModel<'_grist_Filters'> -type DocInfo = RowModel<'_grist_DocInfo'> +type GristViewsSectionField = MetaRowRecord<'_grist_Views_section_field'> +type GristTablesColumn = MetaRowRecord<'_grist_Tables_column'> // Type for filters passed from the client export interface Filter { colRef: number, filter: string } diff --git a/app/server/lib/ExportCSV.ts b/app/server/lib/ExportCSV.ts index f13230c4..46972175 100644 --- a/app/server/lib/ExportCSV.ts +++ b/app/server/lib/ExportCSV.ts @@ -74,7 +74,7 @@ export async function makeCSVFromTable( } // Look up the table to make a CSV from. - const tables = activeDoc.docData.getTable('_grist_Tables')!; + const tables = activeDoc.docData.getMetaTable('_grist_Tables'); const tableRef = tables.findRow('tableId', tableId); if (tableRef === 0) { diff --git a/app/server/lib/GranularAccess.ts b/app/server/lib/GranularAccess.ts index 4270bbfb..925b3f4f 100644 --- a/app/server/lib/GranularAccess.ts +++ b/app/server/lib/GranularAccess.ts @@ -265,10 +265,10 @@ export class GranularAccess implements GranularAccessForBundle { // Create a tmpDocData with just the tables we care about, then update docActions to it. const tmpDocData: DocData = new DocData( (tableId) => { throw new Error("Unexpected DocData fetch"); }, { - _grist_Tables: this._docData.getTable('_grist_Tables')!.getTableDataAction(), - _grist_Tables_column: this._docData.getTable('_grist_Tables_column')!.getTableDataAction(), - _grist_ACLResources: this._docData.getTable('_grist_ACLResources')!.getTableDataAction(), - _grist_ACLRules: this._docData.getTable('_grist_ACLRules')!.getTableDataAction(), + _grist_Tables: this._docData.getMetaTable('_grist_Tables').getTableDataAction(), + _grist_Tables_column: this._docData.getMetaTable('_grist_Tables_column').getTableDataAction(), + _grist_ACLResources: this._docData.getMetaTable('_grist_ACLResources').getTableDataAction(), + _grist_ACLRules: this._docData.getMetaTable('_grist_ACLRules').getTableDataAction(), }); for (const da of docActions) { tmpDocData.receiveAction(da); diff --git a/app/server/lib/OnDemandActions.ts b/app/server/lib/OnDemandActions.ts index 8dcd5831..78d2bd95 100644 --- a/app/server/lib/OnDemandActions.ts +++ b/app/server/lib/OnDemandActions.ts @@ -22,8 +22,8 @@ export interface OnDemandStorage { */ export class OnDemandActions { - private _tablesMeta: TableData = this._docData.getTable('_grist_Tables')!; - private _columnsMeta: TableData = this._docData.getTable('_grist_Tables_column')!; + private _tablesMeta: TableData = this._docData.getMetaTable('_grist_Tables'); + private _columnsMeta: TableData = this._docData.getMetaTable('_grist_Tables_column'); constructor(private _storage: OnDemandStorage, private _docData: DocData) {} diff --git a/app/server/lib/Triggers.ts b/app/server/lib/Triggers.ts index a7fd05ed..5dd354ba 100644 --- a/app/server/lib/Triggers.ts +++ b/app/server/lib/Triggers.ts @@ -138,8 +138,8 @@ export class DocTriggers { } // Happens on doc creation while processing InitNewDoc action. const triggersTable = docData.getMetaTable("_grist_Triggers"); - const getTableId = docData.getMetaTable("_grist_Tables").getMetaRowPropFunc("tableId"); - this._getColId = docData.getMetaTable("_grist_Tables_column").getMetaRowPropFunc("colId"); + const getTableId = docData.getMetaTable("_grist_Tables").getRowPropFunc("tableId"); + this._getColId = docData.getMetaTable("_grist_Tables_column").getRowPropFunc("colId"); const triggersByTableRef = _.groupBy(triggersTable.getRecords(), "tableRef"); const triggersByTableId: Array<[string, Trigger[]]> = [];