use fresher node and debian version (#1255)

This moves to node 22 and debian bookworm, since the versions we've been building and testing with are getting old.

There is some old material kept around for (speaks very quietly) Python 2 (looks around hoping no-one heard) which we continue to support for some long-time users but really really should drop soon.

The changes for the node upgrade were all test related. I did them in a way that shouldn't impair running on older versions of node, and did spot checks for this. This is to give some breathing room for upgrading Grist Lab's grist-saas as follow up work.

.github/workflows/docker_latest.yml

@@ -50,7 +50,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.11]
-        node-version: [18.x]
+        node-version: [22.x]
         image:
           # We build two images, `grist-oss` and `grist`.
           # See https://github.com/gristlabs/grist-core?tab=readme-ov-file#available-docker-images

.github/workflows/main.yml

@@ -18,19 +18,19 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [3.11]
-        node-version: [18.x]
+        node-version: [22.x]
         tests:
           - ':lint:python:client:common:smoke:stubs:'
           - ':server-1-of-2:'
           - ':server-2-of-2:'
-          - ':nbrowser-^[A-G]:'
+          - ':nbrowser-^[A-D]:'
-          - ':nbrowser-^[H-L]:'
+          - ':nbrowser-^[E-L]:'
-          - ':nbrowser-^[M-O]:'
+          - ':nbrowser-^[M-N]:'
-          - ':nbrowser-^[P-S]:'
+          - ':nbrowser-^[O-R]:'
-          - ':nbrowser-^[^A-S]:'
+          - ':nbrowser-^[^A-R]:'
         include:
           - tests: ':lint:python:client:common:smoke:'
-            node-version: 18.x
+            node-version: 22.x
             python-version: '3.10'
     steps:
       - uses: actions/checkout@v3
@@ -125,6 +125,7 @@ jobs:
           ARTIFACT_NAME=logs-$(echo $TESTS | sed 's/[^-a-zA-Z0-9]/_/g')
           echo "Artifact name is '$ARTIFACT_NAME'"
           echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV
+          mkdir -p $TESTDIR
           find $TESTDIR -iname "*.socket" -exec rm {} \;
         env:
           TESTS: ${{ matrix.tests }}

Dockerfile

@@ -10,7 +10,7 @@ FROM scratch AS ext
 ## Javascript build stage
 ################################################################################
 
-FROM node:18-buster AS builder
+FROM node:22-bookworm AS builder
 
 # Install all node dependencies.
 WORKDIR /grist
@@ -45,19 +45,30 @@ RUN \
 ## Python collection stage
 ################################################################################
 
-# Fetch python3.11 and python2.7
+# Fetch python3.11
-FROM python:3.11-slim-buster AS collector
+FROM python:3.11-slim-bookworm AS collector-py3
-
-# Install all python dependencies.
-ADD sandbox/requirements.txt requirements.txt
 ADD sandbox/requirements3.txt requirements3.txt
+RUN \
+  pip3 install -r requirements3.txt
+
+# Fetch <shame>python2.7</shame>
+# This is to support users with old documents.
+# If you have documents with python2.7 formulas, try switching
+# to python3 in the document settings. It'll probably work fine!
+# And we'll be forced to turn off python2 support eventually,
+# the workarounds needed to keep it are getting silly.
+# It doesn't exist in recent Debian, so we need to reach back
+# to buster.
+FROM python:2.7-slim-buster AS collector-py2
+ADD sandbox/requirements.txt requirements.txt
 RUN \
   apt update && \
   apt install -y --no-install-recommends python2 python-pip python-setuptools \
   build-essential libxml2-dev libxslt-dev python-dev zlib1g-dev && \
   pip2 install wheel && \
   pip2 install -r requirements.txt && \
-  pip3 install -r requirements3.txt
+  pip2 install six && \
+  find /usr/lib -iname "libffi.so.6*" -exec cp {} /usr/local/lib \;
 
 ################################################################################
 ## Sandbox collection stage
@@ -66,6 +77,8 @@ RUN \
 # Fetch gvisor-based sandbox. Note, to enable it to run within default
 # unprivileged docker, layers of protection that require privilege have
 # been stripped away, see https://github.com/google/gvisor/issues/4371
+# The sandbox binary is built on buster, but remains compatible with recent
+# Debian.
 FROM docker.io/gristlabs/gvisor-unprivileged:buster AS sandbox
 
 ################################################################################
@@ -73,7 +86,7 @@ FROM docker.io/gristlabs/gvisor-unprivileged:buster AS sandbox
 ################################################################################
 
 # Now, start preparing final image.
-FROM node:18-buster-slim
+FROM node:22-bookworm-slim
 
 # Install libexpat1, libsqlite3-0 for python3 library binary dependencies.
 # Install pgrep for managing gvisor processes.
@@ -91,13 +104,23 @@ COPY --from=builder /grist/node_modules /grist/node_modules
 COPY --from=builder /grist/_build /grist/_build
 COPY --from=builder /grist/static /grist/static-built
 
-# Copy python files.
+# Copy python2 files.
-COPY --from=collector /usr/bin/python2.7 /usr/bin/python2.7
+COPY --from=collector-py2 /usr/bin/python2.7 /usr/bin/python2.7
-COPY --from=collector /usr/lib/python2.7 /usr/lib/python2.7
+COPY --from=collector-py2 /usr/lib/python2.7 /usr/lib/python2.7
-COPY --from=collector /usr/local/lib/python2.7 /usr/local/lib/python2.7
+COPY --from=collector-py2 /usr/local/lib/python2.7 /usr/local/lib/python2.7
-COPY --from=collector /usr/local/bin/python3.11 /usr/bin/python3.11
+# Make a small python2 tweak so that material in /usr/local/lib is found.
-COPY --from=collector /usr/local/lib/python3.11 /usr/local/lib/python3.11
+RUN \
-COPY --from=collector /usr/local/lib/libpython3.11.* /usr/local/lib/
+  mkdir /etc/python2.7 && \
+  echo "import sys\nsys.path.append('/usr/local/lib/python2.7/site-packages')" > /etc/python2.7/sitecustomize.py
+# Copy across an older libffi library binary needed by python2.
+# We moved it a bit sleazily to a predictable location to avoid awkward
+# architecture-dependent logic.
+COPY --from=collector-py2 /usr/local/lib/libffi.so.6* /usr/local/lib
+
+# Copy python3 files.
+COPY --from=collector-py3 /usr/local/bin/python3.11 /usr/bin/python3.11
+COPY --from=collector-py3 /usr/local/lib/python3.11 /usr/local/lib/python3.11
+COPY --from=collector-py3 /usr/local/lib/libpython3.11.* /usr/local/lib/
 # Set default to python3
 RUN \
   ln -s /usr/bin/python3.11 /usr/bin/python && \

app/client/components/GristClientSocket.ts

@@ -71,7 +71,13 @@ export class GristClientSocket {
   }
 
   private _createWSSocket() {
-    if (typeof WebSocket !== 'undefined') {
+    // We used to check if WebSocket was defined here, and use it
+    // if so, secure in the fact that we were in the browser and
+    // the browser would pass along cookie information. But recent
+    // node defines WebSocket, so we narrow down this path to when
+    // a global document is defined (window doesn't work because
+    // some tests mock it).
+    if (typeof document !== 'undefined') {
       this._wsSocket = new WebSocket(this._url);
     } else {
       this._wsSocket = new WS(this._url, undefined, this._options);

test/common/NumberFormat.ts

@@ -6,7 +6,17 @@ describe("NumberFormat", function() {
     locale: 'en-US'
   };
 
+  // useGrouping became more nuanced in recent node.
+  // Its old 'true' value may now be 'always' or 'auto'.
+  const useGroupingAlways = buildNumberFormat(
+    {numMode: 'decimal'},
+    defaultDocSettings
+  ).resolvedOptions().useGrouping as boolean|string;
+  const useGroupingAuto = (useGroupingAlways === 'always') ? 'auto' : true;
+
   it("should convert Grist options into Intr.NumberFormat", function() {
+    assert.include([true, 'always'], String(useGroupingAlways));
+
     assert.ownInclude(buildNumberFormat({}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 0,
       maximumFractionDigits: 10,
@@ -17,21 +27,21 @@ describe("NumberFormat", function() {
       minimumFractionDigits: 0,
       maximumFractionDigits: 3,
       style: 'decimal',
-      useGrouping: true,
+      useGrouping: useGroupingAlways,
     });
     assert.ownInclude(buildNumberFormat({numMode: 'percent'}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 0,
       maximumFractionDigits: 0,
       // style: 'percent',  // In node v14.17.0 style is 'decimal' (unclear why)
                             // so we check final formatting instead in this case.
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
     });
     assert.equal(buildNumberFormat({numMode: 'percent'}, defaultDocSettings).format(0.5), '50%');
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'USD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'scientific'}, defaultDocSettings).resolvedOptions(), {
@@ -73,42 +83,42 @@ describe("NumberFormat", function() {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'EUR',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'en-NZ'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'NZD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'de-CH'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'CHF',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'es-AR'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'ARS',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'zh-TW'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'TWD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'en-AU'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'AUD',
     });
   });

test/nbrowser/Importer.ts

@@ -19,6 +19,10 @@ describe('Importer', function() {
     // have tests go faster. Each successful test case should leave the document unchanged.
     if (!docUrl || !await gu.testCurrentUrl(docUrl)) {
       const session = await gu.session().teamSite.login();
+      // TODO: tests check colors literally, so need to be in
+      // light theme - but calling gu.setGristTheme results in
+      // some problems so right now if you are a dev you just
+      // need to run these tests in light mode, sorry.
       await session.tempDoc(cleanup, 'Hello.grist');
       docUrl = await driver.getCurrentUrl();
     }
@@ -450,9 +454,9 @@ describe('Importer', function() {
       assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true);
 
       // Check that the merge field select button has a red outline.
-      assert.equal(
+      assert.match(
         await driver.find('.test-importer-merge-fields-select').getCssValue('border'),
-        '1px solid rgb(208, 2, 27)'
+        /solid rgb\(208, 2, 27\)/
       );
 
       // Select a merge field, and check that the red outline is gone.
@@ -461,9 +465,9 @@ describe('Importer', function() {
         '.test-multi-select-menu .test-multi-select-menu-option',
         /Name/
       ).click();
-      assert.equal(
+      assert.match(
         await driver.find('.test-importer-merge-fields-select').getCssValue('border'),
-        '1px solid rgb(217, 217, 217)'
+        /solid rgb\(217, 217, 217\)/
       );
       // Hide dropdown
       await gu.sendKeys(Key.ESCAPE);
@@ -584,9 +588,9 @@ describe('Importer', function() {
       await driver.findContent('.test-importer-source', /UploadedData2Extended.csv/).click();
 
       // Check that it failed, and that the merge fields select button is outlined in red.
-      assert.equal(
+      assert.match(
         await driver.find('.test-importer-merge-fields-select').getCssValue('border'),
-        '1px solid rgb(208, 2, 27)'
+        /solid rgb\(208, 2, 27\)/
       );
       assert.equal(
         await driver.find('.test-importer-source-selected .test-importer-from').getText(),

test/nbrowser/gristUtils.ts

@@ -1464,6 +1464,7 @@ export function revertChanges(test: () => Promise<void>, invariant: () => any =
 export async function redo(optCount: number = 1, optTimeout?: number) {
   for (let i = 0; i < optCount; ++i) {
     await driver.find('.test-redo').doClick();
+    await waitForServer(optTimeout);
   }
   await waitForServer(optTimeout);
 }
@@ -2781,11 +2782,11 @@ export function addSamplesForSuite(includeTutorial = false) {
 }
 
 export async function openAccountMenu() {
-  await driver.findWait('.test-dm-account', 1000).click();
+  await driver.findWait('.test-dm-account', 2000).click();
   // Since the AccountWidget loads orgs and the user data asynchronously, the menu
   // can expand itself causing the click to land on a wrong button.
   await waitForServer();
-  await driver.findWait('.test-site-switcher-org', 1000);
+  await driver.findWait('.test-site-switcher-org', 2000);
   await driver.sleep(250);  // There's still some jitter (scroll-bar? other user accounts?)
 }
 

test/nbrowser/importerTestUtils.ts

@@ -35,7 +35,14 @@ export const getPreviewDiffCellValues = stackWrapFunc(async (cols: number[], row
 
 // Helper that waits for the diff preview to finish loading.
 export const waitForDiffPreviewToLoad = stackWrapFunc(async (): Promise<void> => {
+  await gu.waitForServer();
   await driver.wait(() => driver.find('.test-importer-preview').isPresent(), 5000);
+  await gu.waitToPass(async () => {
+    const preview = (await getPreviewDiffCellValues([0], [1]))[0];
+    if (preview[0] === undefined && preview[1] === undefined) {
+      throw new Error('sometimes data is a little slow to show up?');
+    }
+  }, 2000);
 });
 
 // Helper that gets the list of visible column matching rows to the left of the preview.

test/server/lib/DocApi.ts

@@ -76,6 +76,8 @@ function makeConfig(username: string): AxiosRequestConfig {
 }
 
 describe('DocApi', function () {
+  const webhooksTestPort = Number(process.env.WEBHOOK_TEST_PORT || 34365);
+
   this.timeout(30000);
   testUtils.setTmpLogLevel('error');
   let oldEnv: testUtils.EnvironmentSnapshot;
@@ -121,7 +123,7 @@ describe('DocApi', function () {
       homeUrl = serverUrl = home.serverUrl;
       hasHomeApi = true;
     });
-    testDocApi();
+    testDocApi({webhooksTestPort});
   });
 
   describe('With GRIST_ANON_PLAYGROUND disabled', async () => {
@@ -157,7 +159,7 @@ describe('DocApi', function () {
         homeUrl = serverUrl = home.serverUrl;
         hasHomeApi = true;
       });
-      testDocApi();
+      testDocApi({webhooksTestPort});
     });
 
     describe('behind a reverse-proxy', function () {
@@ -206,7 +208,7 @@ describe('DocApi', function () {
 
         after(() => tearDown(proxy, [home, docs]));
 
-        testDocApi();
+        testDocApi({webhooksTestPort});
       });
 
       async function testCompareDocs(proxy: TestServerReverseProxy, home: TestServer) {
@@ -261,7 +263,7 @@ describe('DocApi', function () {
         serverUrl = docs.serverUrl;
         hasHomeApi = false;
       });
-      testDocApi();
+      testDocApi({webhooksTestPort});
     });
   }
 
@@ -323,7 +325,10 @@ describe('DocApi', function () {
 });
 
 // Contains the tests. This is where you want to add more test.
-function testDocApi() {
+function testDocApi(settings: {
+  webhooksTestPort: number,
+}) {
+  const { webhooksTestPort } = settings;
   let chimpy: AxiosRequestConfig, kiwi: AxiosRequestConfig,
     charon: AxiosRequestConfig, nobody: AxiosRequestConfig, support: AxiosRequestConfig;
 
@@ -3478,13 +3483,20 @@ function testDocApi() {
   });
 
   describe('webhooks related endpoints', async function () {
-    const serving: Serving = await serveSomething(app => {
+    let serving: Serving;
-      app.use(express.json());
+    before(async function () {
-      app.post('/200', ({body}, res) => {
+      serving = await serveSomething(app => {
-        res.sendStatus(200);
+        app.use(express.json());
-        res.end();
+        app.post('/200', ({body}, res) => {
-      });
+          res.sendStatus(200);
-    }, webhooksTestPort);
+          res.end();
+        });
+      }, webhooksTestPort);
+    });
+
+    after(async function () {
+      await serving.shutdown();
+    });
 
     /*
       Regression test for old _subscribe endpoint. /docs/{did}/webhooks should be used instead to subscribe
@@ -3577,7 +3589,8 @@ function testDocApi() {
           }
         }]
       },
-        403, /Column not found notExisting/);
+        // this check was previously just wrong, was the test not running somehow??
+        404, /Column not found "notExisting"/);
 
     });
 
@@ -5385,8 +5398,6 @@ async function getWorkspaceId(api: UserAPIImpl, name: string) {
   return workspaces.find((w) => w.name === name)!.id;
 }
 
-const webhooksTestPort = Number(process.env.WEBHOOK_TEST_PORT || 34365);
-
 async function setupDataDir(dir: string) {
   // we'll be serving Hello.grist content for various document ids, so let's make copies of it in
   // tmpDir

test/server/lib/ProxyAgent.ts

@@ -84,15 +84,17 @@ describe("ProxyAgent", function () {
     it("should report error when proxy fails", async function() {
       // if the proxy isn't listening, fetches produces error messages.
       await testProxyServer.dispose();
+      // Error message depends a little on node version.
       const logMessages2 = await captureLog('warn', async () => {
-        await assert.isRejected(testFetch('/200'), /ECONNREFUSED/);
+        await assert.isRejected(testFetch('/200'), /(request.*failed)|(ECONNREFUSED)/);
-        await assert.isRejected(testFetch('/404'), /ECONNREFUSED/);
+        await assert.isRejected(testFetch('/404'), /(request.*failed)|(ECONNREFUSED)/);
       });
 
       // We rely on "ProxyAgent error" message to detect issues with the proxy server.
+      // Error message depends a little on node version.
       assertMatchArray(logMessages2, [
-        /warn: ProxyAgent error.*ECONNREFUSED/,
+        /warn: ProxyAgent error.*((request.*failed)|(ECONNREFUSED)|(AggregateError))/,
-        /warn: ProxyAgent error.*ECONNREFUSED/,
+        /warn: ProxyAgent error.*((request.*failed)|(ECONNREFUSED)|(AggregateError))/,
       ]);
     });
   });

test/server/lib/UnhandledErrors.ts

@@ -51,7 +51,8 @@ describe('UnhandledErrors', function() {
         }, 1000, 100);
 
         // We expect the server to be dead now.
-        await assert.isRejected(fetch(`${server.serverUrl}/status`), /failed.*ECONNREFUSED/);
+        // Error message depends a little on node version.
+        await assert.isRejected(fetch(`${server.serverUrl}/status`), /(request.*failed)|(ECONNREFUSED)/);
 
       } finally {
         await server.stop();