(core) Parse string cell values in Doc API and Imports

Summary:
- Adds a function `parseUserAction` for parsing strings in UserActions to `ValueParser.ts`
- Adds a boolean option `parseStrings` to use `parseUserAction` in `ActiveDoc.applyUserActions`, off by default.
- Uses `parseStrings` by default in DocApi (set `?noparse=true` in a request to disable) when adding/updating records through the `/data` or `/records` endpoints or in general with the `/apply` endpoint.
- Uses `parseStrings` for various actions in `ActiveDocImport`. Since most types are parsed in Python before these actions are constructed, this only affects references, which still look like errors in the import preview. Importing references can also easily still run into more complicated problems discussed in https://grist.slack.com/archives/C0234CPPXPA/p1639514844028200

Test Plan:
- Added tests to DocApi to compare behaviour with and without string parsing.
- Added a new browser test, fixture doc, and fixture CSV to test importing a file containing references.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3183

app/server/lib/ActiveDoc.ts

@@ -41,6 +41,7 @@ import {schema, SCHEMA_VERSION} from 'app/common/schema';
 import {MetaRowRecord} from 'app/common/TableData';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
+import {parseUserAction} from 'app/common/ValueParser';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
 import {GristDocAPI} from 'app/plugin/GristAPI';
 import {compileAclFormula} from 'app/server/lib/ACLFormula';
@@ -1373,6 +1374,11 @@ export class ActiveDoc extends EventEmitter {
     this._log.debug(docSession, "_applyUserActions(%s, %s)", client, shortDesc(actions));
     this._inactivityTimer.ping();     // The doc is in active use; ping it to stay open longer.
 
+    if (options.parseStrings) {
+      actions = actions.map(ua => parseUserAction(ua, this.docData!));
+      this._log.debug(docSession, "_applyUserActions(%s, %s) (after parsing)", client, shortDesc(actions));
+    }
+
     if (options?.bestEffort) {
       actions = await this._granularAccess.prefilterUserActions(docSession, actions);
     }

app/server/lib/ActiveDocImport.ts

@@ -9,7 +9,7 @@ import {ApplyUAResult, DataSourceTransformed, ImportOptions, ImportResult, Impor
         TransformRule,
         TransformRuleMap} from 'app/common/ActiveDocAPI';
 import {ApiError} from 'app/common/ApiError';
-import {BulkColValues, CellValue, fromTableDataAction, TableRecordValue} from 'app/common/DocActions';
+import {BulkColValues, CellValue, fromTableDataAction, TableRecordValue, UserAction} from 'app/common/DocActions';
 import * as gutil from 'app/common/gutil';
 import {DocStateComparison} from 'app/common/UserAPI';
 import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI';
@@ -312,7 +312,7 @@ export class ActiveDocImport {
       const ruleCanBeApplied = (transformRule != null) &&
                                _.difference(transformRule.sourceCols, hiddenTableColIds).length === 0;
       await this._activeDoc.applyUserActions(docSession,
-        [["ReplaceTableData", hiddenTableId, rowIdColumn, columnValues]]);
+        [["ReplaceTableData", hiddenTableId, rowIdColumn, columnValues]], {parseStrings: true});
 
       // data parsed and put into hiddenTableId
       // For preview_table (isHidden) do GenImporterView to make views and formulas and cols
@@ -438,7 +438,8 @@ export class ActiveDocImport {
     }
 
     await this._activeDoc.applyUserActions(docSession,
-        [['BulkAddRecord', destTableId, gutil.arrayRepeat(hiddenTableData.id.length, null), columnData]]);
+      [['BulkAddRecord', destTableId, gutil.arrayRepeat(hiddenTableData.id.length, null), columnData]],
+      {parseStrings: true});
 
     return destTableId;
   }
@@ -518,17 +519,17 @@ export class ActiveDocImport {
     }
 
     // We no longer need the temporary import table, so remove it.
-    await this._activeDoc.applyUserActions(docSession, [['RemoveTable', hiddenTableId]]);
+    const actions: UserAction[] = [['RemoveTable', hiddenTableId]];
 
     if (updatedRecordIds.length > 0) {
-      await this._activeDoc.applyUserActions(docSession,
-        [['BulkUpdateRecord', destTableId, updatedRecordIds, updatedRecords]]);
+      actions.push(['BulkUpdateRecord', destTableId, updatedRecordIds, updatedRecords]);
     }
 
     if (numNewRecords > 0) {
-      await this._activeDoc.applyUserActions(docSession,
-        [['BulkAddRecord', destTableId, gutil.arrayRepeat(numNewRecords, null), newRecords]]);
+      actions.push(['BulkAddRecord', destTableId, gutil.arrayRepeat(numNewRecords, null), newRecords]);
     }
+
+    await this._activeDoc.applyUserActions(docSession, actions, {parseStrings: true});
   }
 
   /**

app/server/lib/DocApi.ts

@@ -2,7 +2,7 @@ import { createEmptyActionSummary } from "app/common/ActionSummary";
 import { ApiError } from 'app/common/ApiError';
 import { BrowserSettings } from "app/common/BrowserSettings";
 import {
-  BulkColValues, CellValue, ColValues, fromTableDataAction, TableColValues, TableRecordValue,
+  BulkColValues, ColValues, fromTableDataAction, TableColValues, TableRecordValue,
 } from 'app/common/DocActions';
 import {isRaisedException} from "app/common/gristTypes";
 import { arrayRepeat, isAffirmative } from "app/common/gutil";
@@ -142,7 +142,8 @@ export class DocWorkerApi {
 
     // Apply user actions to a document.
     this._app.post('/api/docs/:docId/apply', canEdit, withDoc(async (activeDoc, req, res) => {
-      res.json(await activeDoc.applyUserActions(docSessionFromRequest(req), req.body));
+      const parseStrings = !isAffirmative(req.query.noparse);
+      res.json(await activeDoc.applyUserActions(docSessionFromRequest(req), req.body, {parseStrings}));
     }));
 
     async function getTableData(activeDoc: ActiveDoc, req: RequestWithLogin) {
@@ -252,14 +253,9 @@ export class DocWorkerApi {
     async function addRecords(
       req: RequestWithLogin, activeDoc: ActiveDoc, count: number, columnValues: BulkColValues
     ): Promise<number[]> {
-      const tableId = req.params.tableId;
-      const colNames = Object.keys(columnValues);
       // user actions expect [null, ...] as row ids
       const rowIds = arrayRepeat(count, null);
-      const sandboxRes = await handleSandboxError(tableId, colNames, activeDoc.applyUserActions(
-        docSessionFromRequest(req),
-        [['BulkAddRecord', tableId, rowIds, columnValues]]));
-      return sandboxRes.retValues[0];
+      return addOrUpdateRecords(req, activeDoc, columnValues, rowIds, 'BulkAddRecord');
     }
 
     function areSameFields(records: Array<Types.Record | Types.NewRecord>) {
@@ -367,13 +363,24 @@ export class DocWorkerApi {
     // Update records identified by rowIds. Any invalid id fails
     // the request and returns a 400 error code.
     async function updateRecords(
-      req: RequestWithLogin, activeDoc: ActiveDoc, columnValues: {[colId: string]: CellValue[]}, rowIds: number[]
+      req: RequestWithLogin, activeDoc: ActiveDoc, columnValues: BulkColValues, rowIds: number[]
+    ) {
+      await addOrUpdateRecords(req, activeDoc, columnValues, rowIds, 'BulkUpdateRecord');
+    }
+
+    async function addOrUpdateRecords(
+      req: RequestWithLogin, activeDoc: ActiveDoc,
+      columnValues: BulkColValues, rowIds: (number | null)[],
+      actionType: 'BulkUpdateRecord' | 'BulkAddRecord'
     ) {
       const tableId = req.params.tableId;
       const colNames = Object.keys(columnValues);
-      await handleSandboxError(tableId, colNames, activeDoc.applyUserActions(
+      const sandboxRes = await handleSandboxError(tableId, colNames, activeDoc.applyUserActions(
         docSessionFromRequest(req),
-        [['BulkUpdateRecord', tableId, rowIds, columnValues]]));
+        [[actionType, tableId, rowIds, columnValues]],
+        {parseStrings: !isAffirmative(req.query.noparse)},
+      ));
+      return sandboxRes.retValues[0];
     }
 
     // Update records given in column format