(core) Implement exported functions without relying on ActiveDoc.docData

Summary:
For grist-static, we want to the data engine to be able to call external/exported JS functions directly,
rather than via the node 'server' living in another thread which requires synchronous communication hackery.

As a step in that direction, this diff changes the exported functions that we care about (guessColInfo and convertFromColumn)
to just using the top-level functions instead of relying on fields in ActiveDoc, namely docData.

For guessColInfo, this is done by directly passing the small amount of metadata that was previously retrieved from the DocData.

For convertFromColumn, disentangling DocData is a lot more complicated, so instead we construct a fresh DocData object using
the required metadata tables which are now passed in by the data engine.

Test Plan: Existing tests

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3913

app/common/ValueConverter.ts

@@ -10,6 +10,7 @@ import {
   ValueParser
 } from 'app/common/ValueParser';
 import {CellValue, GristObjCode} from 'app/plugin/GristData';
+import { TableDataActionSet } from "./DocActions";
 
 
 /**
@@ -192,22 +193,26 @@ export function createConverter(formatter: BaseFormatter, parser: ValueParser) {
  * The higher order function separates docData (passed by ActiveDoc)
  * from the arguments passed to call_external in Python.
  */
-export function convertFromColumn(docData: DocData) {
+export function convertFromColumn(
-  return function(
+  metaTables: TableDataActionSet,
-    sourceColRef: number,
+  sourceColRef: number,
-    type: string,
+  type: string,
-    widgetOpts: string,
+  widgetOpts: string,
-    visibleColRef: number,
+  visibleColRef: number,
-    values: ReadonlyArray<CellValue>,
+  values: ReadonlyArray<CellValue>,
-    displayColValues?: ReadonlyArray<CellValue>,
+  displayColValues?: ReadonlyArray<CellValue>,
-  ): CellValue[] {
+): CellValue[] {
-    const formatter = createFullFormatterFromDocData(docData, sourceColRef);
+  const docData = new DocData(
-    const parser = createParserRaw(
+    (_tableId) => { throw new Error("Unexpected DocData fetch"); },
-      ...createParserOrFormatterArgumentsRaw(docData, type, widgetOpts, visibleColRef)
+    metaTables,
-    );
+  );
-    const converter = createConverter(formatter, parser);
+
-    return convertValues(converter, values, displayColValues || values);
+  const formatter = createFullFormatterFromDocData(docData, sourceColRef);
-  };
+  const parser = createParserRaw(
+    ...createParserOrFormatterArgumentsRaw(docData, type, widgetOpts, visibleColRef)
+  );
+  const converter = createConverter(formatter, parser);
+  return convertValues(converter, values, displayColValues || values);
 }
 
 export function convertValues(

app/server/lib/ActiveDoc.ts

@@ -79,7 +79,7 @@ import {UIRowId} from 'app/common/UIRowId';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {Document as APIDocument, DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
 import {convertFromColumn} from 'app/common/ValueConverter';
-import {guessColInfoWithDocData} from 'app/common/ValueGuesser';
+import {guessColInfo} from 'app/common/ValueGuesser';
 import {parseUserAction} from 'app/common/ValueParser';
 import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
 import {Document} from 'app/gen-server/entity/Document';
@@ -2637,10 +2637,8 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
       sandboxOptions: {
         exports: {
           request: (key: string, args: SandboxRequest) => this._requests.handleSingleRequestWithCache(key, args),
-          guessColInfo: (values: Array<string | null>) =>
+          guessColInfo,
-            guessColInfoWithDocData(values, this.docData!),
+          convertFromColumn,
-          convertFromColumn: (...args: Parameters<ReturnType<typeof convertFromColumn>>) =>
-            convertFromColumn(this.docData!)(...args)
         }
       },
     });

sandbox/grist/useractions.py

@@ -139,7 +139,7 @@ def _make_clean_col_info(col_info, col_id=None):
   return ret
 
 
-def guess_col_info(values):
+def guess_col_info(values, doc_model):
   """
   Returns a pair col_info, values
   where col_info is a dict which may contain a type and widgetOptions
@@ -154,7 +154,12 @@ def guess_col_info(values):
     # Use the exported guessColInfo if we're connected to JS
     from sandbox import default_sandbox
     if default_sandbox:
-      guess = default_sandbox.call_external("guessColInfo", values)
+      doc_info = doc_model.doc_info.lookupOne()
+      try:
+        doc_settings = json.loads(doc_info.documentSettings)
+      except ValueError:
+        doc_settings = {}
+      guess = default_sandbox.call_external("guessColInfo", values, doc_settings, doc_info.timezone)
       # When the result doesn't contain `values`, that means the guessed type is Text
       # so there was nothing to convert.
       values = guess.get("values", values)
@@ -906,7 +911,7 @@ class UserActions(object):
         # Guess the type when it starts out as Any. We unfortunately need to update the column
         # separately for type conversion, to recompute type-specific defaults
         # before they are used in formula->data conversion.
-        col_info, values = guess_col_info(values)
+        col_info, values = guess_col_info(values, self._docmodel)
         # If the values are all blank (None or empty string) leave the column empty
         if not col_info:
           return values
@@ -1555,8 +1560,20 @@ class UserActions(object):
     if src_col.displayCol:
       display_col = table.get_column(src_col.displayCol.colId)
       display_values = [encode_object(display_col.raw_get(r)) for r in row_ids]
+    meta_table_data = {
+      meta_table_id: actions.get_action_repr(
+        self._engine.fetch_table(meta_table_id, formulas=False)
+      )
+      for meta_table_id in [
+        "_grist_DocInfo",
+        "_grist_Tables",
+        "_grist_Tables_column",
+        "_grist_Views_section_field"
+      ]
+    }
     converted_values = call_external(
       "convertFromColumn",
+      meta_table_data,
       src_col.id,
       typ,
       widgetOptions,