(core) updates from grist-core

.github/workflows/fly-build.yml

@@ -23,7 +23,8 @@ jobs:
       - name: Build and export Docker image
         id: docker-build
         run: >
-          docker build -t grist-core:preview . &&
+          ./buildtools/checkout-ext-directory.sh grist-ee &&
+          docker build -t grist-core:preview . --build-context ext=ext &&
           docker image save grist-core:preview -o grist-core.tar
       - name: Save PR information
         run: |

app/client/apiconsole.ts

@@ -293,6 +293,25 @@ function initialize(appModel: AppModel) {
 
 function requestInterceptor(request: SwaggerUI.Request) {
   delete request.headers.Authorization;
+  const url = new URL(request.url);
+  // Swagger will use this request interceptor for several kinds of
+  // requests, such as requesting the API YAML spec from Github:
+  //
+  //      Function to intercept remote definition, "Try it out",
+  //      and OAuth 2.0 requests.
+  //
+  //    https://swagger.io/docs/open-source-tools/swagger-ui/usage/configuration/
+  //
+  // We want to ensure that only "Try it out" requests have XHR, so
+  // that they pass a same origin request, even if they're not GET,
+  // HEAD, or OPTIONS. "Try it out" requests are the requests to the
+  // same origin.
+  if (url.origin === window.origin) {
+    // Without this header, unauthenticated multipart POST requests
+    // (i.e. file uploads) would fail in the API console. We want those
+    // requests to succeed.
+    request.headers['X-Requested-With'] = 'XMLHttpRequest';
+  }
   return request;
 }
 

app/server/lib/BootProbes.ts

@@ -6,7 +6,7 @@ import { GristServer } from 'app/server/lib/GristServer';
 import * as express from 'express';
 import WS from 'ws';
 import fetch from 'node-fetch';
-import { DEFAULT_SESSION_SECRET } from 'app/server/lib/coreCreator';
+import { DEFAULT_SESSION_SECRET } from 'app/server/lib/ICreate';
 
 /**
  * Self-diagnostics useful when installing Grist.

app/server/lib/GranularAccess.ts

@@ -113,8 +113,6 @@ const SPECIAL_ACTIONS = new Set(['InitNewDoc',
                                  'FillTransformRuleColIds',
                                  'TransformAndFinishImport',
                                  'AddView',
-                                 'CopyFromColumn',
-                                 'ConvertFromColumn',
                                  'AddHiddenColumn',
                                  'RespondToRequests',
                                 ]);
@@ -132,9 +130,7 @@ const OK_ACTIONS = new Set(['Calculate', 'UpdateCurrentTime']);
 // Only add an action to OTHER_RECOGNIZED_ACTIONS if you know access control
 // has been handled for it, or it is clear that access control can be done
 // by looking at the Create/Update/Delete permissions for the DocActions it
-// will create. For example, at the time of writing CopyFromColumn should
-// not be here, since it could read a column the user is not supposed to
-// have access rights to, and it is not handled specially.
+// will create.
 const OTHER_RECOGNIZED_ACTIONS = new Set([
   // Data actions.
   'AddRecord',
@@ -149,6 +145,11 @@ const OTHER_RECOGNIZED_ACTIONS = new Set([
   'AddOrUpdateRecord',
   'BulkAddOrUpdateRecord',
 
+  // Certain column actions are handled specially because of reads that
+  // don't fit the pattern of data actions.
+  'ConvertFromColumn',
+  'CopyFromColumn',
+
   // Groups of actions.
   'ApplyDocActions',
   'ApplyUndoActions',
@@ -818,7 +819,7 @@ export class GranularAccess implements GranularAccessForBundle {
     // Checks are in no particular order.
     await this._checkSimpleDataActions(docSession, actions);
     await this._checkForSpecialOrSurprisingActions(docSession, actions);
-    await this._checkPossiblePythonFormulaModification(docSession, actions);
+    await this._checkIfNeedsEarlySchemaPermission(docSession, actions);
     await this._checkDuplicateTableAccess(docSession, actions);
     await this._checkAddOrUpdateAccess(docSession, actions);
   }
@@ -912,7 +913,14 @@ export class GranularAccess implements GranularAccessForBundle {
    */
   public needEarlySchemaPermission(a: UserAction|DocAction): boolean {
     const name = a[0] as string;
-    if (name === 'ModifyColumn' || name === 'SetDisplayFormula') {
+    if (name === 'ModifyColumn' || name === 'SetDisplayFormula' ||
+        // ConvertFromColumn and CopyFromColumn are hard to reason
+        // about, especially since they appear in bundles with other
+        // actions. We throw up our hands a bit here, and just make
+        // sure the user has schema permissions. Today, in Grist, that
+        // gives a lot of power. If this gets narrowed down in future,
+        // we'll have to rethink this.
+        name === 'ConvertFromColumn' || name === 'CopyFromColumn') {
       return true;
     } else if (isDataAction(a)) {
       const tableId = getTableId(a);
@@ -1362,7 +1370,6 @@ export class GranularAccess implements GranularAccessForBundle {
     }
 
     await this._assertOnlyBundledWithSimpleDataActions(ADD_OR_UPDATE_RECORD_ACTIONS, actions);
-
     // Check for read access, and that we're not touching metadata.
     await applyToActionsRecursively(actions, async (a) => {
       if (!isAddOrUpdateRecordAction(a)) { return; }
@@ -1392,12 +1399,15 @@ export class GranularAccess implements GranularAccessForBundle {
     });
   }
 
-  private async _checkPossiblePythonFormulaModification(docSession: OptDocSession, actions: UserAction[]) {
+  private async _checkIfNeedsEarlySchemaPermission(docSession: OptDocSession, actions: UserAction[]) {
     // If changes could include Python formulas, then user must have
     // +S before we even consider passing these to the data engine.
     // Since we don't track rule or schema changes at this stage, we
     // approximate with the user's access rights at beginning of
     // bundle.
+    // We also check for +S in scenarios that are hard to break down
+    // in a more granular way, for example ConvertFromColumn and
+    // CopyFromColumn.
     if (scanActionsRecursively(actions, (a) => this.needEarlySchemaPermission(a))) {
       await this._assertSchemaAccess(docSession);
     }

app/server/lib/ICreate.ts

@@ -13,6 +13,29 @@ import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
 import {SqliteVariant} from 'app/server/lib/SqliteCommon';
 import {ITelemetry} from 'app/server/lib/Telemetry';
 
+// In the past, the session secret was used as an additional
+// protection passed on to expressjs-session for security when
+// generating session IDs, in order to make them less guessable.
+// Quoting the upstream documentation,
+//
+//     Using a secret that cannot be guessed will reduce the ability
+//     to hijack a session to only guessing the session ID (as
+//     determined by the genid option).
+//
+//   https://expressjs.com/en/resources/middleware/session.html
+//
+// However, since this change,
+//
+//   https://github.com/gristlabs/grist-core/commit/24ce54b586e20a260376a9e3d5b6774e3fa2b8b8#diff-d34f5357f09d96e1c2ba63495da16aad7bc4c01e7925ab1e96946eacd1edb094R121-R124
+//
+// session IDs are now completely randomly generated in a cryptographically
+// secure way, so there is no danger of session IDs being guessable.
+// This makes the value of the session secret less important. The only
+// concern is that changing the secret will invalidate existing
+// sessions and force users to log in again.
+export const DEFAULT_SESSION_SECRET =
+  'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
+
 export interface ICreate {
 
   Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
@@ -72,6 +95,15 @@ export interface ICreateTelemetryOptions {
   create(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry|undefined;
 }
 
+/**
+ * This function returns a `create` object that defines various core
+ * aspects of a Grist installation, such as what kind of billing or
+ * sandbox to use, if any.
+ *
+ * The intended use of this function is to initialise Grist with
+ * different settings and providers, to facilitate different editions
+ * such as standard, enterprise or cloud-hosted.
+ */
 export function makeSimpleCreator(opts: {
   deploymentType: GristDeploymentType,
   sessionSecret?: string,
@@ -116,11 +148,7 @@ export function makeSimpleCreator(opts: {
       return createSandbox(opts.sandboxFlavor || 'unsandboxed', options);
     },
     sessionSecret() {
-      const secret = process.env.GRIST_SESSION_SECRET || sessionSecret;
-      if (!secret) {
-        throw new Error('need GRIST_SESSION_SECRET');
-      }
-      return secret;
+      return process.env.GRIST_SESSION_SECRET || sessionSecret || DEFAULT_SESSION_SECRET;
     },
     async configure() {
       for (const s of storage || []) {

app/server/lib/coreCreator.ts

@@ -3,14 +3,8 @@ import { checkMinIOBucket, checkMinIOExternalStorage,
 import { makeSimpleCreator } from 'app/server/lib/ICreate';
 import { Telemetry } from 'app/server/lib/Telemetry';
 
-export const DEFAULT_SESSION_SECRET =
-  'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
-
 export const makeCoreCreator = () => makeSimpleCreator({
   deploymentType: 'core',
-  // This can and should be overridden by GRIST_SESSION_SECRET
-  // (or generated randomly per install, like grist-omnibus does).
-  sessionSecret: DEFAULT_SESSION_SECRET,
   storage: [
     {
       name: 'minio',

buildtools/.grist-ee-version

@@ -1 +1 @@
-0.9.5
+0.9.6

documentation/grist-data-format.md

@@ -124,6 +124,9 @@ Note that this combination of rules allows tables and column names to be valid i
 
 ## Value Types
 
+> [!WARNING]
+> This section is out of date.
+
 The format supports a number of data types. Some types have a short representation (e.g. `Numeric` as a JSON `number`, and `Text` as a JSON `string`), but all types have an explicit representation as well.
 
 The explicit representation of a value is an array `[typeCode, args...]`. The first member of the array is a string code that defines the type of the value. The rest of the elements are arguments used to construct the actual value.
@@ -131,6 +134,7 @@ The explicit representation of a value is an array `[typeCode, args...]`. The fi
 The following table lists currently supported types and their short and explicit representations.
 
 | **Type Name**    | **Short Repr** | **[Type Code, Args...]**      | **Description**                                                                                                               |
+|------------------|----------------|-------------------------------|-------------------------------------------------------------------------------------------------------------------------------|
 | `Numeric`        | `number`*      | `['n',number]`                | double-precision floating point number                                                                                        |
 | `Text`           | `string`*      | `['s',string]`                | Unicode string                                                                                                                |
 | `Bool`           | `bool`*        | `['b',bool]`                  | Boolean value (true or false)                                                                                                 |

static/locales/en.client.json

@@ -1633,5 +1633,37 @@
         "Number of Calls": "Number of Calls",
         "Table ID": "Table ID",
         "Total Time (s)": "Total Time (s)"
+    },
+    "DocTutorial": {
+        "Click to expand": "Click to expand",
+        "Do you want to restart the tutorial? All progress will be lost.": "Do you want to restart the tutorial? All progress will be lost.",
+        "End tutorial": "End tutorial",
+        "Finish": "Finish",
+        "Next": "Next",
+        "Previous": "Previous",
+        "Restart": "Restart"
+    },
+    "OnboardingCards": {
+        "3 minute video tour": "3 minute video tour",
+        "Complete our basics tutorial": "Complete our basics tutorial",
+        "Complete the tutorial": "Complete the tutorial",
+        "Learn the basic of reference columns, linked widgets, column types, & cards.": "Learn the basic of reference columns, linked widgets, column types, & cards."
+    },
+    "OnboardingPage": {
+        "Back": "Back",
+        "Discover Grist in 3 minutes": "Discover Grist in 3 minutes",
+        "Go hands-on with the Grist Basics tutorial": "Go hands-on with the Grist Basics tutorial",
+        "Go to the tutorial!": "Go to the tutorial!",
+        "Next step": "Next step",
+        "Skip step": "Skip step",
+        "Skip tutorial": "Skip tutorial",
+        "Tell us who you are": "Tell us who you are",
+        "Type here": "Type here",
+        "Welcome": "Welcome",
+        "What brings you to Grist (you can select multiple)?": "What brings you to Grist (you can select multiple)?",
+        "What is your role?": "What is your role?",
+        "What organization are you with?": "What organization are you with?",
+        "Your organization": "Your organization",
+        "Your role": "Your role"
     }
 }

test/server/lib/GranularAccess.ts

@@ -457,6 +457,58 @@ describe('GranularAccess', function() {
     ]);
   });
 
+  it('respects SCHEMA_EDIT when converting a column', async () => {
+    // Initially, schema flag defaults to ON for editor.
+    await freshDoc();
+    await owner.applyUserActions(docId, [
+      ['AddTable', 'Table1', [{id: 'A', type: 'Int'},
+                              {id: 'B', type: 'Int'},
+                              {id: 'C', type: 'Int'}]],
+      ['AddRecord', '_grist_ACLResources', -1, {tableId: 'Table1', colIds: 'C'}],
+      // Add at least one access rule. Otherwise the test would succeed
+      // trivially, via shortcuts in place when the GranularAccess
+      // hasNuancedAccess test returns false. If there are no access
+      // rules present, editors can make any edit. Once a granular access
+      // rule is present, editors lose some rights that are simply too
+      // hard to compute or we haven't gotten around to.
+      ['AddRecord', '_grist_ACLRules', null, {
+        resource: -1, aclFormula: 'user.Access == OWNER', permissionsText: '-R',
+      }],
+      ['AddRecord', 'Table1', null, {A: 1234, B: 1234}],
+    ]);
+
+    // Make a transformation as editor.
+    await editor.applyUserActions(docId, [
+      ['AddColumn', 'Table1', 'gristHelper_Converted', {type: 'Text', isFormula: false, visibleCol: 0, formula: ''}],
+      ['AddColumn', 'Table1', 'gristHelper_Transform',
+       {type: 'Text', isFormula: true, visibleCol: 0, formula: 'rec.gristHelper_Converted'}],
+      ["ConvertFromColumn", "Table1", "A", "gristHelper_Converted", "Text", "", 0],
+      ["CopyFromColumn", "Table1", "gristHelper_Transform", "A", "{}"],
+    ]);
+
+    // Now turn off schema flag for editor.
+    await owner.applyUserActions(docId, [
+      ['AddRecord', '_grist_ACLResources', -1, {tableId: '*', colIds: '*'}],
+      ['AddRecord', '_grist_ACLRules', null, {
+        resource: -1, aclFormula: 'user.Access == EDITOR', permissionsText: '-S',
+      }],
+    ]);
+
+    // Now prepare another transformation.
+    const transformation = [
+      ['AddColumn', 'Table1', 'gristHelper_Converted2', {type: 'Text', isFormula: false, visibleCol: 0, formula: ''}],
+      ['AddColumn', 'Table1', 'gristHelper_Transform2',
+       {type: 'Text', isFormula: true, visibleCol: 0, formula: 'rec.gristHelper_Converted2'}],
+      ["ConvertFromColumn", "Table1", "B", "gristHelper_Converted2", "Text", "", 0],
+      ["CopyFromColumn", "Table1", "gristHelper_Transform", "B", "{}"],
+    ];
+    // Should fail for editor.
+    await assert.isRejected(editor.applyUserActions(docId, transformation),
+                            /Blocked by full structure access rules/);
+    // Should go through if run as owner.
+    await assert.isFulfilled(owner.applyUserActions(docId, transformation));
+  });
+
   async function applyTransformation(colToHide: string) {
     await freshDoc();
     await owner.applyUserActions(docId, [
@@ -906,12 +958,12 @@ describe('GranularAccess', function() {
 
     await assert.isRejected(editor.applyUserActions(docId, [
       ['CopyFromColumn', 'Data1', 'A', 'B', {}],
-    ]), /need uncomplicated access/);
+    ]), /Blocked by full structure access rules/);
 
     await assert.isRejected(editor.applyUserActions(docId, [
       ['RenameColumn', 'Data1', 'B', 'B'],
       ['CopyFromColumn', 'Data1', 'A', 'B', {}],
-    ]), /need uncomplicated access/);
+    ]), /Blocked by full structure access rules/);
 
     assert.deepEqual(await editor.getDocAPI(docId).getRows('Data1'), {
       id: [ 1, 2 ],