(core) Move guessing logic for column types to run in node once for all columns.

Summary:
Previously, columns of type Any were created and modified one by one by reusing
the "empty column" logic from the data engine. This copies that logic to Node,
and sets the type of all columns together, to create them with the correct type
in the AddTable call.

This makes imports about twice faster (when slowness is due to many columns),
but doesn't address all cases where individual handling of columns causes slowness.

Test Plan: Added a test case for the new helper function.

Reviewers: alexmojaki

Reviewed By: alexmojaki

Subscribers: alexmojaki

Differential Revision: https://phab.getgrist.com/D3427

app/common/ValueGuesser.ts

@@ -1,9 +1,12 @@
+import {CellValue} from 'app/common/DocActions';
 import {DocData} from 'app/common/DocData';
 import {DocumentSettings} from 'app/common/DocumentSettings';
+import {isObject} from 'app/common/gristTypes';
 import {countIf} from 'app/common/gutil';
 import {NumberFormatOptions} from 'app/common/NumberFormat';
 import NumberParse from 'app/common/NumberParse';
 import {dateTimeWidgetOptions, guessDateFormat} from 'app/common/parseDate';
+import {MetaRowRecord} from 'app/common/TableData';
 import {createFormatter} from 'app/common/ValueFormatter';
 import {createParserRaw, ValueParser} from 'app/common/ValueParser';
 import * as moment from 'moment-timezone';
@@ -14,10 +17,17 @@ interface GuessedColInfo {
 }
 
 export interface GuessResult {
-  values?: any[];
+  values?: CellValue[];
   colInfo: GuessedColInfo;
 }
 
+type ColMetadata = Partial<MetaRowRecord<'_grist_Tables_column'>>;
+
+export interface GuessColMetadata {
+  values: CellValue[];
+  colMetadata?: ColMetadata;    // omitted if no changes are proposed.
+}
+
 /**
  * Class for guessing if an array of values should be interpreted as a specific column type.
  * T is the type of values that strings should be parsed to and is stored in the column.
@@ -169,3 +179,29 @@ export function guessColInfo(
     {colInfo: {type: 'Text'}}
   );
 }
+
+/**
+ * Guess column info for a new column, returning the metadata suitable for using with AddTable or
+ * AddColumn user actions. In particular, widgetOptions, if any, are returned as a JSON string.
+ * Will suggest turning the column to an empty one if all the values are empty (null or "").
+ */
+export function guessColInfoForImports(values: CellValue[], docData: DocData): GuessColMetadata {
+  if (values.every(v => (v === null || v === ''))) {
+    // Suggest empty column.
+    return {values, colMetadata: {type: 'Any', isFormula: true, formula: ''}};
+  }
+  if (values.some(isObject)) {
+    // Suggest no changes.
+    return {values};
+  }
+  const strValues = values.map(v => (v === null || typeof v === 'string' ? v : String(v)));
+  const guessed = guessColInfoWithDocData(strValues, docData);
+  values = guessed.values || values;
+
+  const opts = guessed.colInfo.widgetOptions;
+  const colMetadata: ColMetadata = {...guessed.colInfo, widgetOptions: opts && JSON.stringify(opts)};
+  if (!colMetadata.widgetOptions) {
+    delete colMetadata.widgetOptions;     // Omit widgetOptions unless it is actually valid JSON.
+  }
+  return {values, colMetadata};
+}

app/server/lib/ActiveDocImport.ts

@@ -12,6 +12,7 @@ import {ApiError} from 'app/common/ApiError';
 import {BulkColValues, CellValue, fromTableDataAction, TableRecordValue, UserAction} from 'app/common/DocActions';
 import * as gutil from 'app/common/gutil';
 import {DocStateComparison} from 'app/common/UserAPI';
+import {guessColInfoForImports} from 'app/common/ValueGuesser';
 import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI';
 import {GristColumn, GristTable} from 'app/plugin/GristTable';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
@@ -303,7 +304,7 @@ export class ActiveDocImport {
       const origTableName = table.table_name ? table.table_name : '';
       const transformRule = transformRuleMap && transformRuleMap.hasOwnProperty(origTableName) ?
         transformRuleMap[origTableName] : null;
-      const columnMetadata = cleanColumnMetadata(table.column_metadata);
+      const columnMetadata = cleanColumnMetadata(table.column_metadata, table.table_data, this._activeDoc);
       const result: ApplyUAResult = await this._activeDoc.applyUserActions(docSession,
         [["AddTable", hiddenTableName, columnMetadata]]);
       const retValue: AddTableRetValue = result.retValues[0];
@@ -758,17 +759,24 @@ function getMergeFunction({type}: MergeStrategy): MergeFunction {
  * Tweak the column metadata used in the AddTable action.
  * If `columns` is populated with non-blank column ids, adds labels to all
  * columns using the values set for the column ids.
- * Ensure that columns of type Any start out as formula columns, i.e. empty columns,
- * so that type guessing is triggered when new data is added.
+ * For columns of type Any, guess the type and parse data according to it, or mark as empty
+ * formula columns when they should be empty.
  */
-function cleanColumnMetadata(columns: GristColumn[]) {
-  return columns.map(c => {
+function cleanColumnMetadata(columns: GristColumn[], tableData: unknown[][], activeDoc: ActiveDoc) {
+  return columns.map((c, index) => {
     const newCol: any = {...c};
     if (c.id) {
       newCol.label = c.id;
     }
     if (c.type === "Any") {
-      newCol.isFormula = true;
+      // If import logic left it to us to decide on column type, then use our guessing logic to
+      // pick a suitable type and widgetOptions, and to convert values to it.
+      const origValues = tableData[index] as CellValue[];
+      const {values, colMetadata} = guessColInfoForImports(origValues, activeDoc.docData!);
+      tableData[index] = values;
+      if (colMetadata) {
+        Object.assign(newCol, colMetadata);
+      }
     }
     return newCol;
   });