(core) updates from grist-core

.github/workflows/fly-build.yml

@@ -23,7 +23,8 @@ jobs:
       - name: Build and export Docker image
         id: docker-build
         run: >
-          docker build -t grist-core:preview . &&
+          ./buildtools/checkout-ext-directory.sh grist-ee &&
+          docker build -t grist-core:preview . --build-context ext=ext &&
           docker image save grist-core:preview -o grist-core.tar
       - name: Save PR information
         run: |

app/client/apiconsole.ts

@@ -293,6 +293,25 @@ function initialize(appModel: AppModel) {
 
 function requestInterceptor(request: SwaggerUI.Request) {
   delete request.headers.Authorization;
+  const url = new URL(request.url);
+  // Swagger will use this request interceptor for several kinds of
+  // requests, such as requesting the API YAML spec from Github:
+  //
+  //      Function to intercept remote definition, "Try it out",
+  //      and OAuth 2.0 requests.
+  //
+  //    https://swagger.io/docs/open-source-tools/swagger-ui/usage/configuration/
+  //
+  // We want to ensure that only "Try it out" requests have XHR, so
+  // that they pass a same origin request, even if they're not GET,
+  // HEAD, or OPTIONS. "Try it out" requests are the requests to the
+  // same origin.
+  if (url.origin === window.origin) {
+    // Without this header, unauthenticated multipart POST requests
+    // (i.e. file uploads) would fail in the API console. We want those
+    // requests to succeed.
+    request.headers['X-Requested-With'] = 'XMLHttpRequest';
+  }
   return request;
 }
 

app/server/lib/BootProbes.ts

@@ -6,7 +6,7 @@ import { GristServer } from 'app/server/lib/GristServer';
 import * as express from 'express';
 import WS from 'ws';
 import fetch from 'node-fetch';
-import { DEFAULT_SESSION_SECRET } from 'app/server/lib/coreCreator';
+import { DEFAULT_SESSION_SECRET } from 'app/server/lib/ICreate';
 
 /**
  * Self-diagnostics useful when installing Grist.

app/server/lib/GranularAccess.ts

@@ -113,8 +113,6 @@ const SPECIAL_ACTIONS = new Set(['InitNewDoc',
                                  'FillTransformRuleColIds',
                                  'TransformAndFinishImport',
                                  'AddView',
-                                 'CopyFromColumn',
-                                 'ConvertFromColumn',
                                  'AddHiddenColumn',
                                  'RespondToRequests',
                                 ]);
@@ -132,9 +130,7 @@ const OK_ACTIONS = new Set(['Calculate', 'UpdateCurrentTime']);
 // Only add an action to OTHER_RECOGNIZED_ACTIONS if you know access control
 // has been handled for it, or it is clear that access control can be done
 // by looking at the Create/Update/Delete permissions for the DocActions it
-// will create. For example, at the time of writing CopyFromColumn should
-// not be here, since it could read a column the user is not supposed to
-// have access rights to, and it is not handled specially.
+// will create.
 const OTHER_RECOGNIZED_ACTIONS = new Set([
   // Data actions.
   'AddRecord',
@@ -149,6 +145,11 @@ const OTHER_RECOGNIZED_ACTIONS = new Set([
   'AddOrUpdateRecord',
   'BulkAddOrUpdateRecord',
 
+  // Certain column actions are handled specially because of reads that
+  // don't fit the pattern of data actions.
+  'ConvertFromColumn',
+  'CopyFromColumn',
+
   // Groups of actions.
   'ApplyDocActions',
   'ApplyUndoActions',
@@ -818,7 +819,7 @@ export class GranularAccess implements GranularAccessForBundle {
     // Checks are in no particular order.
     await this._checkSimpleDataActions(docSession, actions);
     await this._checkForSpecialOrSurprisingActions(docSession, actions);
-    await this._checkPossiblePythonFormulaModification(docSession, actions);
+    await this._checkIfNeedsEarlySchemaPermission(docSession, actions);
     await this._checkDuplicateTableAccess(docSession, actions);
     await this._checkAddOrUpdateAccess(docSession, actions);
   }
@@ -912,7 +913,14 @@ export class GranularAccess implements GranularAccessForBundle {
    */
   public needEarlySchemaPermission(a: UserAction|DocAction): boolean {
     const name = a[0] as string;
-    if (name === 'ModifyColumn' || name === 'SetDisplayFormula') {
+    if (name === 'ModifyColumn' || name === 'SetDisplayFormula' ||
+        // ConvertFromColumn and CopyFromColumn are hard to reason
+        // about, especially since they appear in bundles with other
+        // actions. We throw up our hands a bit here, and just make
+        // sure the user has schema permissions. Today, in Grist, that
+        // gives a lot of power. If this gets narrowed down in future,
+        // we'll have to rethink this.
+        name === 'ConvertFromColumn' || name === 'CopyFromColumn') {
       return true;
     } else if (isDataAction(a)) {
       const tableId = getTableId(a);
@@ -1362,7 +1370,6 @@ export class GranularAccess implements GranularAccessForBundle {
     }
 
     await this._assertOnlyBundledWithSimpleDataActions(ADD_OR_UPDATE_RECORD_ACTIONS, actions);
-
     // Check for read access, and that we're not touching metadata.
     await applyToActionsRecursively(actions, async (a) => {
       if (!isAddOrUpdateRecordAction(a)) { return; }
@@ -1392,12 +1399,15 @@ export class GranularAccess implements GranularAccessForBundle {
     });
   }
 
-  private async _checkPossiblePythonFormulaModification(docSession: OptDocSession, actions: UserAction[]) {
+  private async _checkIfNeedsEarlySchemaPermission(docSession: OptDocSession, actions: UserAction[]) {
     // If changes could include Python formulas, then user must have
     // +S before we even consider passing these to the data engine.
     // Since we don't track rule or schema changes at this stage, we
     // approximate with the user's access rights at beginning of
     // bundle.
+    // We also check for +S in scenarios that are hard to break down
+    // in a more granular way, for example ConvertFromColumn and
+    // CopyFromColumn.
     if (scanActionsRecursively(actions, (a) => this.needEarlySchemaPermission(a))) {
       await this._assertSchemaAccess(docSession);
     }

app/server/lib/ICreate.ts

@@ -13,6 +13,29 @@ import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
 import {SqliteVariant} from 'app/server/lib/SqliteCommon';
 import {ITelemetry} from 'app/server/lib/Telemetry';
 
+// In the past, the session secret was used as an additional
+// protection passed on to expressjs-session for security when
+// generating session IDs, in order to make them less guessable.
+// Quoting the upstream documentation,
+//
+//     Using a secret that cannot be guessed will reduce the ability
+//     to hijack a session to only guessing the session ID (as
+//     determined by the genid option).
+//
+//   https://expressjs.com/en/resources/middleware/session.html
+//
+// However, since this change,
+//
+//   https://github.com/gristlabs/grist-core/commit/24ce54b586e20a260376a9e3d5b6774e3fa2b8b8#diff-d34f5357f09d96e1c2ba63495da16aad7bc4c01e7925ab1e96946eacd1edb094R121-R124
+//
+// session IDs are now completely randomly generated in a cryptographically
+// secure way, so there is no danger of session IDs being guessable.
+// This makes the value of the session secret less important. The only
+// concern is that changing the secret will invalidate existing
+// sessions and force users to log in again.
+export const DEFAULT_SESSION_SECRET =
+  'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
+
 export interface ICreate {
 
   Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
@@ -72,6 +95,15 @@ export interface ICreateTelemetryOptions {
   create(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry|undefined;
 }
 
+/**
+ * This function returns a `create` object that defines various core
+ * aspects of a Grist installation, such as what kind of billing or
+ * sandbox to use, if any.
+ *
+ * The intended use of this function is to initialise Grist with
+ * different settings and providers, to facilitate different editions
+ * such as standard, enterprise or cloud-hosted.
+ */
 export function makeSimpleCreator(opts: {
   deploymentType: GristDeploymentType,
   sessionSecret?: string,
@@ -116,11 +148,7 @@ export function makeSimpleCreator(opts: {
       return createSandbox(opts.sandboxFlavor || 'unsandboxed', options);
     },
     sessionSecret() {
-      const secret = process.env.GRIST_SESSION_SECRET || sessionSecret;
-      if (!secret) {
-        throw new Error('need GRIST_SESSION_SECRET');
-      }
-      return secret;
+      return process.env.GRIST_SESSION_SECRET || sessionSecret || DEFAULT_SESSION_SECRET;
     },
     async configure() {
       for (const s of storage || []) {

app/server/lib/coreCreator.ts

@@ -3,14 +3,8 @@ import { checkMinIOBucket, checkMinIOExternalStorage,
 import { makeSimpleCreator } from 'app/server/lib/ICreate';
 import { Telemetry } from 'app/server/lib/Telemetry';
 
-export const DEFAULT_SESSION_SECRET =
-  'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
-
 export const makeCoreCreator = () => makeSimpleCreator({
   deploymentType: 'core',
-  // This can and should be overridden by GRIST_SESSION_SECRET
-  // (or generated randomly per install, like grist-omnibus does).
-  sessionSecret: DEFAULT_SESSION_SECRET,
   storage: [
     {
       name: 'minio',

buildtools/.grist-ee-version

@@ -1 +1 @@
-0.9.5
+0.9.6

documentation/grist-data-format.md

@@ -124,28 +124,32 @@ Note that this combination of rules allows tables and column names to be valid i
 
 ## Value Types
 
+> [!WARNING]
+> This section is out of date.
+
 The format supports a number of data types. Some types have a short representation (e.g. `Numeric` as a JSON `number`, and `Text` as a JSON `string`), but all types have an explicit representation as well.
 
 The explicit representation of a value is an array `[typeCode, args...]`. The first member of the array is a string code that defines the type of the value. The rest of the elements are arguments used to construct the actual value.
 
 The following table lists currently supported types and their short and explicit representations.
 
-| **Type Name** | **Short Repr** | **[Type Code, Args...]** | **Description** |
-| `Numeric` | `number`* | `['n',number]` | double-precision floating point number |
-| `Text` | `string`* | `['s',string]` | Unicode string |
-| `Bool` | `bool`* | `['b',bool]` | Boolean value (true or false) |
-| `Null` | `null`* | `null` | Null value (no special explicit representation) |
-| `Int` | `number` | `['i',number]` | 32-bit integer |
-| `Date` | `number` | `['d',number]` | Calendar date, represented as seconds since Epoch to 00:00 UTC on that date. |
-| `DateTime` | `number` | `['D',number]` | Instance in time, represented as seconds since Epoch |
-| `Reference` | `number` | `['R',number]`  | Identifier of a record in a table. |
-| `ReferenceList` | | `['L',number,...]` | List of record identifiers |
-| `Choice` | `string` | `['C',string]` | Unicode string selected from a list of choices. |
-| `PositionNumber` | `number` | `['P',number]` | a double used to order records relative to each other. |
-| `Image` | | `['I',string]` | Binary data representing an image, encoded as base64 |
-| `List` | | `['l',values,...]` | List of values of any type. |
-| `JSON` | | `['J',object]` | JSON-serializable object |
-| `Error` | | `['E',string,string?,value?]` | Exception, with first argument exception type, second an optional message, and optionally a third containing additional info. |
+| **Type Name**    | **Short Repr** | **[Type Code, Args...]**      | **Description**                                                                                                               |
+|------------------|----------------|-------------------------------|-------------------------------------------------------------------------------------------------------------------------------|
+| `Numeric`        | `number`*      | `['n',number]`                | double-precision floating point number                                                                                        |
+| `Text`           | `string`*      | `['s',string]`                | Unicode string                                                                                                                |
+| `Bool`           | `bool`*        | `['b',bool]`                  | Boolean value (true or false)                                                                                                 |
+| `Null`           | `null`*        | `null`                        | Null value (no special explicit representation)                                                                               |
+| `Int`            | `number`       | `['i',number]`                | 32-bit integer                                                                                                                |
+| `Date`           | `number`       | `['d',number]`                | Calendar date, represented as seconds since Epoch to 00:00 UTC on that date.                                                  |
+| `DateTime`       | `number`       | `['D',number]`                | Instance in time, represented as seconds since Epoch                                                                          |
+| `Reference`      | `number`       | `['R',number]`                | Identifier of a record in a table.                                                                                            |
+| `ReferenceList`  |                | `['L',number,...]`            | List of record identifiers                                                                                                    |
+| `Choice`         | `string`       | `['C',string]`                | Unicode string selected from a list of choices.                                                                               |
+| `PositionNumber` | `number`       | `['P',number]`                | a double used to order records relative to each other.                                                                        |
+| `Image`          |                | `['I',string]`                | Binary data representing an image, encoded as base64                                                                          |
+| `List`           |                | `['l',values,...]`            | List of values of any type.                                                                                                   |
+| `JSON`           |                | `['J',object]`                | JSON-serializable object                                                                                                      |
+| `Error`          |                | `['E',string,string?,value?]` | Exception, with first argument exception type, second an optional message, and optionally a third containing additional info. |
 
 An important goal is to represent data efficiently in the common case. When a value matches the column's type, the short representation is used. For example, in a Numeric column, a Numeric value is represented as a `number`, and in a Date column, a Date value is represented as a `number`.
 

static/locales/en.client.json

@@ -1633,5 +1633,37 @@
         "Number of Calls": "Number of Calls",
         "Table ID": "Table ID",
         "Total Time (s)": "Total Time (s)"
+    },
+    "DocTutorial": {
+        "Click to expand": "Click to expand",
+        "Do you want to restart the tutorial? All progress will be lost.": "Do you want to restart the tutorial? All progress will be lost.",
+        "End tutorial": "End tutorial",
+        "Finish": "Finish",
+        "Next": "Next",
+        "Previous": "Previous",
+        "Restart": "Restart"
+    },
+    "OnboardingCards": {
+        "3 minute video tour": "3 minute video tour",
+        "Complete our basics tutorial": "Complete our basics tutorial",
+        "Complete the tutorial": "Complete the tutorial",
+        "Learn the basic of reference columns, linked widgets, column types, & cards.": "Learn the basic of reference columns, linked widgets, column types, & cards."
+    },
+    "OnboardingPage": {
+        "Back": "Back",
+        "Discover Grist in 3 minutes": "Discover Grist in 3 minutes",
+        "Go hands-on with the Grist Basics tutorial": "Go hands-on with the Grist Basics tutorial",
+        "Go to the tutorial!": "Go to the tutorial!",
+        "Next step": "Next step",
+        "Skip step": "Skip step",
+        "Skip tutorial": "Skip tutorial",
+        "Tell us who you are": "Tell us who you are",
+        "Type here": "Type here",
+        "Welcome": "Welcome",
+        "What brings you to Grist (you can select multiple)?": "What brings you to Grist (you can select multiple)?",
+        "What is your role?": "What is your role?",
+        "What organization are you with?": "What organization are you with?",
+        "Your organization": "Your organization",
+        "Your role": "Your role"
     }
 }

test/server/lib/GranularAccess.ts

@@ -457,6 +457,58 @@ describe('GranularAccess', function() {
     ]);
   });
 
+  it('respects SCHEMA_EDIT when converting a column', async () => {
+    // Initially, schema flag defaults to ON for editor.
+    await freshDoc();
+    await owner.applyUserActions(docId, [
+      ['AddTable', 'Table1', [{id: 'A', type: 'Int'},
+                              {id: 'B', type: 'Int'},
+                              {id: 'C', type: 'Int'}]],
+      ['AddRecord', '_grist_ACLResources', -1, {tableId: 'Table1', colIds: 'C'}],
+      // Add at least one access rule. Otherwise the test would succeed
+      // trivially, via shortcuts in place when the GranularAccess
+      // hasNuancedAccess test returns false. If there are no access
+      // rules present, editors can make any edit. Once a granular access
+      // rule is present, editors lose some rights that are simply too
+      // hard to compute or we haven't gotten around to.
+      ['AddRecord', '_grist_ACLRules', null, {
+        resource: -1, aclFormula: 'user.Access == OWNER', permissionsText: '-R',
+      }],
+      ['AddRecord', 'Table1', null, {A: 1234, B: 1234}],
+    ]);
+
+    // Make a transformation as editor.
+    await editor.applyUserActions(docId, [
+      ['AddColumn', 'Table1', 'gristHelper_Converted', {type: 'Text', isFormula: false, visibleCol: 0, formula: ''}],
+      ['AddColumn', 'Table1', 'gristHelper_Transform',
+       {type: 'Text', isFormula: true, visibleCol: 0, formula: 'rec.gristHelper_Converted'}],
+      ["ConvertFromColumn", "Table1", "A", "gristHelper_Converted", "Text", "", 0],
+      ["CopyFromColumn", "Table1", "gristHelper_Transform", "A", "{}"],
+    ]);
+
+    // Now turn off schema flag for editor.
+    await owner.applyUserActions(docId, [
+      ['AddRecord', '_grist_ACLResources', -1, {tableId: '*', colIds: '*'}],
+      ['AddRecord', '_grist_ACLRules', null, {
+        resource: -1, aclFormula: 'user.Access == EDITOR', permissionsText: '-S',
+      }],
+    ]);
+
+    // Now prepare another transformation.
+    const transformation = [
+      ['AddColumn', 'Table1', 'gristHelper_Converted2', {type: 'Text', isFormula: false, visibleCol: 0, formula: ''}],
+      ['AddColumn', 'Table1', 'gristHelper_Transform2',
+       {type: 'Text', isFormula: true, visibleCol: 0, formula: 'rec.gristHelper_Converted2'}],
+      ["ConvertFromColumn", "Table1", "B", "gristHelper_Converted2", "Text", "", 0],
+      ["CopyFromColumn", "Table1", "gristHelper_Transform", "B", "{}"],
+    ];
+    // Should fail for editor.
+    await assert.isRejected(editor.applyUserActions(docId, transformation),
+                            /Blocked by full structure access rules/);
+    // Should go through if run as owner.
+    await assert.isFulfilled(owner.applyUserActions(docId, transformation));
+  });
+
   async function applyTransformation(colToHide: string) {
     await freshDoc();
     await owner.applyUserActions(docId, [
@@ -906,12 +958,12 @@ describe('GranularAccess', function() {
 
     await assert.isRejected(editor.applyUserActions(docId, [
       ['CopyFromColumn', 'Data1', 'A', 'B', {}],
-    ]), /need uncomplicated access/);
+    ]), /Blocked by full structure access rules/);
 
     await assert.isRejected(editor.applyUserActions(docId, [
       ['RenameColumn', 'Data1', 'B', 'B'],
       ['CopyFromColumn', 'Data1', 'A', 'B', {}],
-    ]), /need uncomplicated access/);
+    ]), /Blocked by full structure access rules/);
 
     assert.deepEqual(await editor.getDocAPI(docId).getRows('Data1'), {
       id: [ 1, 2 ],