diff --git a/app/gen-server/entity/Limit.ts b/app/gen-server/entity/Limit.ts index 1011a98f..eeb40f5e 100644 --- a/app/gen-server/entity/Limit.ts +++ b/app/gen-server/entity/Limit.ts @@ -7,23 +7,23 @@ export class Limit extends BaseEntity { @PrimaryGeneratedColumn() public id: number; - @Column() + @Column({type: Number}) public limit: number; - @Column() + @Column({type: Number}) public usage: number; - @Column() + @Column({type: String}) public type: string; - @Column({name: 'billing_account_id'}) + @Column({name: 'billing_account_id', type: Number}) public billingAccountId: number; @ManyToOne(type => BillingAccount) @JoinColumn({name: 'billing_account_id'}) public billingAccount: BillingAccount; - @Column({name: 'created_at', default: () => "CURRENT_TIMESTAMP"}) + @Column({name: 'created_at', type: nativeValues.dateTimeType, default: () => "CURRENT_TIMESTAMP"}) public createdAt: Date; /** diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index bb4ce879..c37f405e 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -80,7 +80,7 @@ import {guessColInfo} from 'app/common/ValueGuesser'; import {parseUserAction} from 'app/common/ValueParser'; import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer'; import {Document} from 'app/gen-server/entity/Document'; -import {ParseOptions} from 'app/plugin/FileParserAPI'; +import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI'; import {AccessTokenOptions, AccessTokenResult, GristDocAPI} from 'app/plugin/GristAPI'; import {compileAclFormula} from 'app/server/lib/ACLFormula'; import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance'; @@ -113,7 +113,7 @@ import tmp from 'tmp'; import {ActionHistory} from './ActionHistory'; import {ActionHistoryImpl} from './ActionHistoryImpl'; -import {ActiveDocImport} from './ActiveDocImport'; +import {ActiveDocImport, FileImportOptions} from './ActiveDocImport'; import {DocClients} from './DocClients'; import {DocPluginManager} from './DocPluginManager'; import { @@ -773,6 +773,17 @@ export class ActiveDoc extends EventEmitter { await this._activeDocImport.oneStepImport(docSession, uploadInfo); } + /** + * Import data resulting from parsing a file into a new table. + * In normal circumstances this is only used internally. + * It's exposed publicly for use by grist-static which doesn't use the plugin system. + */ + public async importParsedFileAsNewTable( + docSession: OptDocSession, optionsAndData: ParseFileResult, importOptions: FileImportOptions + ): Promise { + return this._activeDocImport.importParsedFileAsNewTable(docSession, optionsAndData, importOptions); + } + /** * This function saves attachments from a given upload and creates an entry for them in the database. * It returns the list of rowIds for the rows created in the _grist_Attachments table. diff --git a/app/server/lib/ActiveDocImport.ts b/app/server/lib/ActiveDocImport.ts index d257c0b6..3e86fcd5 100644 --- a/app/server/lib/ActiveDocImport.ts +++ b/app/server/lib/ActiveDocImport.ts @@ -44,7 +44,7 @@ interface ReferenceDescription { refTableId: string; } -interface FileImportOptions { +export interface FileImportOptions { // Suggested name of the import file. It is sometimes used as a suggested table name, e.g. for csv imports. originalFilename: string; // Containing parseOptions as serialized JSON to pass to the import plugin. @@ -227,71 +227,14 @@ export class ActiveDocImport { } /** - * Imports all files as new tables, using the given transform rules and import options. - * The isHidden flag indicates whether to create temporary hidden tables, or final ones. - */ - private async _importFiles(docSession: OptDocSession, upload: UploadInfo, transforms: TransformRuleMap[], - {parseOptions = {}, mergeOptionMaps = []}: ImportOptions, - isHidden: boolean): Promise { - - // Check that upload size is within the configured limits. - const limit = (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || Infinity; - const totalSize = upload.files.reduce((acc, f) => acc + f.size, 0); - if (totalSize > limit) { - throw new ApiError(`Imported files must not exceed ${gutil.byteString(limit)}`, 413); - } - - // The upload must be within the plugin-accessible directory. Once moved, subsequent calls to - // moveUpload() will return without having to do anything. - if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); } - await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir()); - - const importResult: ImportResult = {options: parseOptions, tables: []}; - for (const [index, file] of upload.files.entries()) { - // If we have a better guess for the file's extension, replace it in origName, to ensure - // that DocPluginManager has access to it to guess the best parser type. - let origName: string = file.origName; - if (file.ext) { - origName = path.basename(origName, path.extname(origName)) + file.ext; - } - const res = await this._importFileAsNewTable(docSession, file.absPath, { - parseOptions, - mergeOptionsMap: mergeOptionMaps[index] || {}, - isHidden, - originalFilename: origName, - uploadFileIndex: index, - transformRuleMap: transforms[index] || {} - }); - if (index === 0) { - // Returned parse options from the first file should be used for all files in one upload. - importResult.options = parseOptions = res.options; - } - importResult.tables.push(...res.tables); - } - return importResult; - } - - /** - * Imports the data stored at tmpPath. - * - * Currently it starts a python parser as a child process - * outside the sandbox, and supports xlsx, csv, and perhaps some other formats. It may - * result in the import of multiple tables, in case of e.g. Excel formats. - * @param {OptDocSession} docSession: Session instance to use for importing. - * @param {String} tmpPath: The path from of the original file. - * @param {FileImportOptions} importOptions: File import options. - * @returns {Promise} with `options` property containing parseOptions as serialized JSON as adjusted - * or guessed by the plugin, and `tables`, which is which is a list of objects with information about - * tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`. + * Import data resulting from parsing a file into a new table. + * In normal circumstances this is only used internally. + * It's exposed publicly for use by grist-static which doesn't use the plugin system. */ - private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string, - importOptions: FileImportOptions): Promise { - const {originalFilename, parseOptions, mergeOptionsMap, isHidden, uploadFileIndex, - transformRuleMap} = importOptions; - log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename); - if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); } - const optionsAndData: ParseFileResult = - await this._activeDoc.docPluginManager.parseFile(tmpPath, originalFilename, parseOptions); + public async importParsedFileAsNewTable( + docSession: OptDocSession, optionsAndData: ParseFileResult, importOptions: FileImportOptions + ): Promise { + const {originalFilename, mergeOptionsMap, isHidden, uploadFileIndex, transformRuleMap} = importOptions; const options = optionsAndData.parseOptions; const parsedTables = optionsAndData.tables; @@ -374,6 +317,76 @@ export class ActiveDocImport { return ({options, tables}); } + /** + * Imports all files as new tables, using the given transform rules and import options. + * The isHidden flag indicates whether to create temporary hidden tables, or final ones. + */ + private async _importFiles(docSession: OptDocSession, upload: UploadInfo, transforms: TransformRuleMap[], + {parseOptions = {}, mergeOptionMaps = []}: ImportOptions, + isHidden: boolean): Promise { + + // Check that upload size is within the configured limits. + const limit = (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || Infinity; + const totalSize = upload.files.reduce((acc, f) => acc + f.size, 0); + if (totalSize > limit) { + throw new ApiError(`Imported files must not exceed ${gutil.byteString(limit)}`, 413); + } + + // The upload must be within the plugin-accessible directory. Once moved, subsequent calls to + // moveUpload() will return without having to do anything. + if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); } + await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir()); + + const importResult: ImportResult = {options: parseOptions, tables: []}; + for (const [index, file] of upload.files.entries()) { + // If we have a better guess for the file's extension, replace it in origName, to ensure + // that DocPluginManager has access to it to guess the best parser type. + let origName: string = file.origName; + if (file.ext) { + origName = path.basename(origName, path.extname(origName)) + file.ext; + } + const res = await this._importFileAsNewTable(docSession, file.absPath, { + parseOptions, + mergeOptionsMap: mergeOptionMaps[index] || {}, + isHidden, + originalFilename: origName, + uploadFileIndex: index, + transformRuleMap: transforms[index] || {} + }); + if (index === 0) { + // Returned parse options from the first file should be used for all files in one upload. + importResult.options = parseOptions = res.options; + } + importResult.tables.push(...res.tables); + } + return importResult; + } + + /** + * Imports the data stored at tmpPath. + * + * Currently it starts a python parser as a child process + * outside the sandbox, and supports xlsx, csv, and perhaps some other formats. It may + * result in the import of multiple tables, in case of e.g. Excel formats. + * @param {OptDocSession} docSession: Session instance to use for importing. + * @param {String} tmpPath: The path from of the original file. + * @param {FileImportOptions} importOptions: File import options. + * @returns {Promise} with `options` property containing parseOptions as serialized JSON as adjusted + * or guessed by the plugin, and `tables`, which is which is a list of objects with information about + * tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`. + */ + private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string, + importOptions: FileImportOptions): Promise { + const {originalFilename, parseOptions} = importOptions; + log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename); + if (!this._activeDoc.docPluginManager) { + throw new Error('no plugin manager available'); + } + const optionsAndData: ParseFileResult = + await this._activeDoc.docPluginManager.parseFile(tmpPath, originalFilename, parseOptions); + return this.importParsedFileAsNewTable(docSession, optionsAndData, importOptions); + } + /** * Imports records from `hiddenTableId` into `destTableId`, transforming the column * values from `hiddenTableId` according to the `transformRule`. Finalizes import when done.