Split out new importFileAsNewTable method for grist-static (#564)

Also add column types to Limit entity to fix errors.

app/gen-server/entity/Limit.ts

@@ -7,23 +7,23 @@ export class Limit extends BaseEntity {
   @PrimaryGeneratedColumn()
   public id: number;
 
-  @Column()
+  @Column({type: Number})
   public limit: number;
 
-  @Column()
+  @Column({type: Number})
   public usage: number;
 
-  @Column()
+  @Column({type: String})
   public type: string;
 
-  @Column({name: 'billing_account_id'})
+  @Column({name: 'billing_account_id', type: Number})
   public billingAccountId: number;
 
   @ManyToOne(type => BillingAccount)
   @JoinColumn({name: 'billing_account_id'})
   public billingAccount: BillingAccount;
 
-  @Column({name: 'created_at', default: () => "CURRENT_TIMESTAMP"})
+  @Column({name: 'created_at', type: nativeValues.dateTimeType, default: () => "CURRENT_TIMESTAMP"})
   public createdAt: Date;
 
   /**

app/server/lib/ActiveDoc.ts

@@ -80,7 +80,7 @@ import {guessColInfo} from 'app/common/ValueGuesser';
 import {parseUserAction} from 'app/common/ValueParser';
 import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
 import {Document} from 'app/gen-server/entity/Document';
-import {ParseOptions} from 'app/plugin/FileParserAPI';
+import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI';
 import {AccessTokenOptions, AccessTokenResult, GristDocAPI} from 'app/plugin/GristAPI';
 import {compileAclFormula} from 'app/server/lib/ACLFormula';
 import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance';
@@ -113,7 +113,7 @@ import tmp from 'tmp';
 
 import {ActionHistory} from './ActionHistory';
 import {ActionHistoryImpl} from './ActionHistoryImpl';
-import {ActiveDocImport} from './ActiveDocImport';
+import {ActiveDocImport, FileImportOptions} from './ActiveDocImport';
 import {DocClients} from './DocClients';
 import {DocPluginManager} from './DocPluginManager';
 import {
@@ -773,6 +773,17 @@ export class ActiveDoc extends EventEmitter {
     await this._activeDocImport.oneStepImport(docSession, uploadInfo);
   }
 
+  /**
+   * Import data resulting from parsing a file into a new table.
+   * In normal circumstances this is only used internally.
+   * It's exposed publicly for use by grist-static which doesn't use the plugin system.
+   */
+  public async importParsedFileAsNewTable(
+    docSession: OptDocSession, optionsAndData: ParseFileResult, importOptions: FileImportOptions
+  ): Promise<ImportResult> {
+    return this._activeDocImport.importParsedFileAsNewTable(docSession, optionsAndData, importOptions);
+  }
+
   /**
    * This function saves attachments from a given upload and creates an entry for them in the database.
    * It returns the list of rowIds for the rows created in the _grist_Attachments table.

app/server/lib/ActiveDocImport.ts

@@ -44,7 +44,7 @@ interface ReferenceDescription {
   refTableId: string;
 }
 
-interface FileImportOptions {
+export interface FileImportOptions {
   // Suggested name of the import file. It is sometimes used as a suggested table name, e.g. for csv imports.
   originalFilename: string;
   // Containing parseOptions as serialized JSON to pass to the import plugin.
@@ -227,71 +227,14 @@ export class ActiveDocImport {
   }
 
   /**
-   * Imports all files as new tables, using the given transform rules and import options.
+   * Import data resulting from parsing a file into a new table.
-   * The isHidden flag indicates whether to create temporary hidden tables, or final ones.
+   * In normal circumstances this is only used internally.
-   */
+   * It's exposed publicly for use by grist-static which doesn't use the plugin system.
-  private async _importFiles(docSession: OptDocSession, upload: UploadInfo, transforms: TransformRuleMap[],
-                             {parseOptions = {}, mergeOptionMaps = []}: ImportOptions,
-                             isHidden: boolean): Promise<ImportResult> {
-
-    // Check that upload size is within the configured limits.
-    const limit = (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || Infinity;
-    const totalSize = upload.files.reduce((acc, f) => acc + f.size, 0);
-    if (totalSize > limit) {
-      throw new ApiError(`Imported files must not exceed ${gutil.byteString(limit)}`, 413);
-    }
-
-    // The upload must be within the plugin-accessible directory. Once moved, subsequent calls to
-    // moveUpload() will return without having to do anything.
-    if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
-    await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir());
-
-    const importResult: ImportResult = {options: parseOptions, tables: []};
-    for (const [index, file] of upload.files.entries()) {
-      // If we have a better guess for the file's extension, replace it in origName, to ensure
-      // that DocPluginManager has access to it to guess the best parser type.
-      let origName: string = file.origName;
-      if (file.ext) {
-        origName = path.basename(origName, path.extname(origName)) + file.ext;
-      }
-      const res = await this._importFileAsNewTable(docSession, file.absPath, {
-        parseOptions,
-        mergeOptionsMap: mergeOptionMaps[index] || {},
-        isHidden,
-        originalFilename: origName,
-        uploadFileIndex: index,
-        transformRuleMap: transforms[index] || {}
-      });
-      if (index === 0) {
-        // Returned parse options from the first file should be used for all files in one upload.
-        importResult.options = parseOptions = res.options;
-      }
-      importResult.tables.push(...res.tables);
-    }
-    return importResult;
-  }
-
-  /**
-   * Imports the data stored at tmpPath.
-   *
-   * Currently it starts a python parser as a child process
-   * outside the sandbox, and supports xlsx, csv, and perhaps some other formats. It may
-   * result in the import of multiple tables, in case of e.g. Excel formats.
-   * @param {OptDocSession} docSession: Session instance to use for importing.
-   * @param {String} tmpPath: The path from of the original file.
-   * @param {FileImportOptions} importOptions: File import options.
-   * @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
-   * or guessed by the plugin, and `tables`, which is which is a list of objects with information about
-   * tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
    */
-  private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string,
+  public async importParsedFileAsNewTable(
-                                      importOptions: FileImportOptions): Promise<ImportResult> {
+    docSession: OptDocSession, optionsAndData: ParseFileResult, importOptions: FileImportOptions
-    const {originalFilename, parseOptions, mergeOptionsMap, isHidden, uploadFileIndex,
+  ): Promise<ImportResult> {
-           transformRuleMap} = importOptions;
+    const {originalFilename, mergeOptionsMap, isHidden, uploadFileIndex, transformRuleMap} = importOptions;
-    log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename);
-    if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
-    const optionsAndData: ParseFileResult =
-      await this._activeDoc.docPluginManager.parseFile(tmpPath, originalFilename, parseOptions);
     const options = optionsAndData.parseOptions;
 
     const parsedTables = optionsAndData.tables;
@@ -374,6 +317,76 @@ export class ActiveDocImport {
     return ({options, tables});
   }
 
+  /**
+   * Imports all files as new tables, using the given transform rules and import options.
+   * The isHidden flag indicates whether to create temporary hidden tables, or final ones.
+   */
+  private async _importFiles(docSession: OptDocSession, upload: UploadInfo, transforms: TransformRuleMap[],
+                             {parseOptions = {}, mergeOptionMaps = []}: ImportOptions,
+                             isHidden: boolean): Promise<ImportResult> {
+
+    // Check that upload size is within the configured limits.
+    const limit = (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || Infinity;
+    const totalSize = upload.files.reduce((acc, f) => acc + f.size, 0);
+    if (totalSize > limit) {
+      throw new ApiError(`Imported files must not exceed ${gutil.byteString(limit)}`, 413);
+    }
+
+    // The upload must be within the plugin-accessible directory. Once moved, subsequent calls to
+    // moveUpload() will return without having to do anything.
+    if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
+    await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir());
+
+    const importResult: ImportResult = {options: parseOptions, tables: []};
+    for (const [index, file] of upload.files.entries()) {
+      // If we have a better guess for the file's extension, replace it in origName, to ensure
+      // that DocPluginManager has access to it to guess the best parser type.
+      let origName: string = file.origName;
+      if (file.ext) {
+        origName = path.basename(origName, path.extname(origName)) + file.ext;
+      }
+      const res = await this._importFileAsNewTable(docSession, file.absPath, {
+        parseOptions,
+        mergeOptionsMap: mergeOptionMaps[index] || {},
+        isHidden,
+        originalFilename: origName,
+        uploadFileIndex: index,
+        transformRuleMap: transforms[index] || {}
+      });
+      if (index === 0) {
+        // Returned parse options from the first file should be used for all files in one upload.
+        importResult.options = parseOptions = res.options;
+      }
+      importResult.tables.push(...res.tables);
+    }
+    return importResult;
+  }
+
+  /**
+   * Imports the data stored at tmpPath.
+   *
+   * Currently it starts a python parser as a child process
+   * outside the sandbox, and supports xlsx, csv, and perhaps some other formats. It may
+   * result in the import of multiple tables, in case of e.g. Excel formats.
+   * @param {OptDocSession} docSession: Session instance to use for importing.
+   * @param {String} tmpPath: The path from of the original file.
+   * @param {FileImportOptions} importOptions: File import options.
+   * @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
+   * or guessed by the plugin, and `tables`, which is which is a list of objects with information about
+   * tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
+   */
+  private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string,
+                                      importOptions: FileImportOptions): Promise<ImportResult> {
+    const {originalFilename, parseOptions} = importOptions;
+    log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename);
+    if (!this._activeDoc.docPluginManager) {
+      throw new Error('no plugin manager available');
+    }
+    const optionsAndData: ParseFileResult =
+      await this._activeDoc.docPluginManager.parseFile(tmpPath, originalFilename, parseOptions);
+    return this.importParsedFileAsNewTable(docSession, optionsAndData, importOptions);
+  }
+
   /**
    * Imports records from `hiddenTableId` into `destTableId`, transforming the column
    * values from `hiddenTableId` according to the `transformRule`. Finalizes import when done.