use fresher node and debian version (#1255)

This moves to node 22 and debian bookworm, since the versions we've been building and testing with are getting old.

There is some old material kept around for (speaks very quietly) Python 2 (looks around hoping no-one heard) which we continue to support for some long-time users but really really should drop soon.

The changes for the node upgrade were all test related. I did them in a way that shouldn't impair running on older versions of node, and did spot checks for this. This is to give some breathing room for upgrading Grist Lab's grist-saas as follow up work.

test/common/NumberFormat.ts

@@ -6,7 +6,17 @@ describe("NumberFormat", function() {
     locale: 'en-US'
   };
 
+  // useGrouping became more nuanced in recent node.
+  // Its old 'true' value may now be 'always' or 'auto'.
+  const useGroupingAlways = buildNumberFormat(
+    {numMode: 'decimal'},
+    defaultDocSettings
+  ).resolvedOptions().useGrouping as boolean|string;
+  const useGroupingAuto = (useGroupingAlways === 'always') ? 'auto' : true;
+
   it("should convert Grist options into Intr.NumberFormat", function() {
+    assert.include([true, 'always'], String(useGroupingAlways));
+
     assert.ownInclude(buildNumberFormat({}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 0,
       maximumFractionDigits: 10,
@@ -17,21 +27,21 @@ describe("NumberFormat", function() {
       minimumFractionDigits: 0,
       maximumFractionDigits: 3,
       style: 'decimal',
-      useGrouping: true,
+      useGrouping: useGroupingAlways,
     });
     assert.ownInclude(buildNumberFormat({numMode: 'percent'}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 0,
       maximumFractionDigits: 0,
       // style: 'percent',  // In node v14.17.0 style is 'decimal' (unclear why)
                             // so we check final formatting instead in this case.
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
     });
     assert.equal(buildNumberFormat({numMode: 'percent'}, defaultDocSettings).format(0.5), '50%');
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'USD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'scientific'}, defaultDocSettings).resolvedOptions(), {
@@ -73,42 +83,42 @@ describe("NumberFormat", function() {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'EUR',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'en-NZ'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'NZD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'de-CH'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'CHF',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'es-AR'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'ARS',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'zh-TW'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'TWD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'en-AU'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'AUD',
     });
   });

test/nbrowser/Importer.ts

@@ -19,6 +19,10 @@ describe('Importer', function() {
     // have tests go faster. Each successful test case should leave the document unchanged.
     if (!docUrl || !await gu.testCurrentUrl(docUrl)) {
       const session = await gu.session().teamSite.login();
+      // TODO: tests check colors literally, so need to be in
+      // light theme - but calling gu.setGristTheme results in
+      // some problems so right now if you are a dev you just
+      // need to run these tests in light mode, sorry.
       await session.tempDoc(cleanup, 'Hello.grist');
       docUrl = await driver.getCurrentUrl();
     }
@@ -450,9 +454,9 @@ describe('Importer', function() {
       assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true);
 
       // Check that the merge field select button has a red outline.
-      assert.equal(
+      assert.match(
         await driver.find('.test-importer-merge-fields-select').getCssValue('border'),
-        '1px solid rgb(208, 2, 27)'
+        /solid rgb\(208, 2, 27\)/
       );
 
       // Select a merge field, and check that the red outline is gone.
@@ -461,9 +465,9 @@ describe('Importer', function() {
         '.test-multi-select-menu .test-multi-select-menu-option',
         /Name/
       ).click();
-      assert.equal(
+      assert.match(
         await driver.find('.test-importer-merge-fields-select').getCssValue('border'),
-        '1px solid rgb(217, 217, 217)'
+        /solid rgb\(217, 217, 217\)/
       );
       // Hide dropdown
       await gu.sendKeys(Key.ESCAPE);
@@ -584,9 +588,9 @@ describe('Importer', function() {
       await driver.findContent('.test-importer-source', /UploadedData2Extended.csv/).click();
 
       // Check that it failed, and that the merge fields select button is outlined in red.
-      assert.equal(
+      assert.match(
         await driver.find('.test-importer-merge-fields-select').getCssValue('border'),
-        '1px solid rgb(208, 2, 27)'
+        /solid rgb\(208, 2, 27\)/
       );
       assert.equal(
         await driver.find('.test-importer-source-selected .test-importer-from').getText(),

test/nbrowser/gristUtils.ts

@@ -1464,6 +1464,7 @@ export function revertChanges(test: () => Promise<void>, invariant: () => any =
 export async function redo(optCount: number = 1, optTimeout?: number) {
   for (let i = 0; i < optCount; ++i) {
     await driver.find('.test-redo').doClick();
+    await waitForServer(optTimeout);
   }
   await waitForServer(optTimeout);
 }
@@ -2781,11 +2782,11 @@ export function addSamplesForSuite(includeTutorial = false) {
 }
 
 export async function openAccountMenu() {
-  await driver.findWait('.test-dm-account', 1000).click();
+  await driver.findWait('.test-dm-account', 2000).click();
   // Since the AccountWidget loads orgs and the user data asynchronously, the menu
   // can expand itself causing the click to land on a wrong button.
   await waitForServer();
-  await driver.findWait('.test-site-switcher-org', 1000);
+  await driver.findWait('.test-site-switcher-org', 2000);
   await driver.sleep(250);  // There's still some jitter (scroll-bar? other user accounts?)
 }
 

test/nbrowser/importerTestUtils.ts

@@ -35,7 +35,14 @@ export const getPreviewDiffCellValues = stackWrapFunc(async (cols: number[], row
 
 // Helper that waits for the diff preview to finish loading.
 export const waitForDiffPreviewToLoad = stackWrapFunc(async (): Promise<void> => {
+  await gu.waitForServer();
   await driver.wait(() => driver.find('.test-importer-preview').isPresent(), 5000);
+  await gu.waitToPass(async () => {
+    const preview = (await getPreviewDiffCellValues([0], [1]))[0];
+    if (preview[0] === undefined && preview[1] === undefined) {
+      throw new Error('sometimes data is a little slow to show up?');
+    }
+  }, 2000);
 });
 
 // Helper that gets the list of visible column matching rows to the left of the preview.

test/server/lib/DocApi.ts

@@ -76,6 +76,8 @@ function makeConfig(username: string): AxiosRequestConfig {
 }
 
 describe('DocApi', function () {
+  const webhooksTestPort = Number(process.env.WEBHOOK_TEST_PORT || 34365);
+
   this.timeout(30000);
   testUtils.setTmpLogLevel('error');
   let oldEnv: testUtils.EnvironmentSnapshot;
@@ -121,7 +123,7 @@ describe('DocApi', function () {
       homeUrl = serverUrl = home.serverUrl;
       hasHomeApi = true;
     });
-    testDocApi();
+    testDocApi({webhooksTestPort});
   });
 
   describe('With GRIST_ANON_PLAYGROUND disabled', async () => {
@@ -157,7 +159,7 @@ describe('DocApi', function () {
         homeUrl = serverUrl = home.serverUrl;
         hasHomeApi = true;
       });
-      testDocApi();
+      testDocApi({webhooksTestPort});
     });
 
     describe('behind a reverse-proxy', function () {
@@ -206,7 +208,7 @@ describe('DocApi', function () {
 
         after(() => tearDown(proxy, [home, docs]));
 
-        testDocApi();
+        testDocApi({webhooksTestPort});
       });
 
       async function testCompareDocs(proxy: TestServerReverseProxy, home: TestServer) {
@@ -261,7 +263,7 @@ describe('DocApi', function () {
         serverUrl = docs.serverUrl;
         hasHomeApi = false;
       });
-      testDocApi();
+      testDocApi({webhooksTestPort});
     });
   }
 
@@ -323,7 +325,10 @@ describe('DocApi', function () {
 });
 
 // Contains the tests. This is where you want to add more test.
-function testDocApi() {
+function testDocApi(settings: {
+  webhooksTestPort: number,
+}) {
+  const { webhooksTestPort } = settings;
   let chimpy: AxiosRequestConfig, kiwi: AxiosRequestConfig,
     charon: AxiosRequestConfig, nobody: AxiosRequestConfig, support: AxiosRequestConfig;
 
@@ -3478,13 +3483,20 @@ function testDocApi() {
   });
 
   describe('webhooks related endpoints', async function () {
-    const serving: Serving = await serveSomething(app => {
-      app.use(express.json());
-      app.post('/200', ({body}, res) => {
-        res.sendStatus(200);
-        res.end();
-      });
-    }, webhooksTestPort);
+    let serving: Serving;
+    before(async function () {
+      serving = await serveSomething(app => {
+        app.use(express.json());
+        app.post('/200', ({body}, res) => {
+          res.sendStatus(200);
+          res.end();
+        });
+      }, webhooksTestPort);
+    });
+
+    after(async function () {
+      await serving.shutdown();
+    });
 
     /*
       Regression test for old _subscribe endpoint. /docs/{did}/webhooks should be used instead to subscribe
@@ -3577,7 +3589,8 @@ function testDocApi() {
           }
         }]
       },
-        403, /Column not found notExisting/);
+        // this check was previously just wrong, was the test not running somehow??
+        404, /Column not found "notExisting"/);
 
     });
 
@@ -5385,8 +5398,6 @@ async function getWorkspaceId(api: UserAPIImpl, name: string) {
   return workspaces.find((w) => w.name === name)!.id;
 }
 
-const webhooksTestPort = Number(process.env.WEBHOOK_TEST_PORT || 34365);
-
 async function setupDataDir(dir: string) {
   // we'll be serving Hello.grist content for various document ids, so let's make copies of it in
   // tmpDir

test/server/lib/ProxyAgent.ts

@@ -84,15 +84,17 @@ describe("ProxyAgent", function () {
     it("should report error when proxy fails", async function() {
       // if the proxy isn't listening, fetches produces error messages.
       await testProxyServer.dispose();
+      // Error message depends a little on node version.
       const logMessages2 = await captureLog('warn', async () => {
-        await assert.isRejected(testFetch('/200'), /ECONNREFUSED/);
-        await assert.isRejected(testFetch('/404'), /ECONNREFUSED/);
+        await assert.isRejected(testFetch('/200'), /(request.*failed)|(ECONNREFUSED)/);
+        await assert.isRejected(testFetch('/404'), /(request.*failed)|(ECONNREFUSED)/);
       });
 
       // We rely on "ProxyAgent error" message to detect issues with the proxy server.
+      // Error message depends a little on node version.
       assertMatchArray(logMessages2, [
-        /warn: ProxyAgent error.*ECONNREFUSED/,
-        /warn: ProxyAgent error.*ECONNREFUSED/,
+        /warn: ProxyAgent error.*((request.*failed)|(ECONNREFUSED)|(AggregateError))/,
+        /warn: ProxyAgent error.*((request.*failed)|(ECONNREFUSED)|(AggregateError))/,
       ]);
     });
   });

test/server/lib/UnhandledErrors.ts

@@ -51,7 +51,8 @@ describe('UnhandledErrors', function() {
         }, 1000, 100);
 
         // We expect the server to be dead now.
-        await assert.isRejected(fetch(`${server.serverUrl}/status`), /failed.*ECONNREFUSED/);
+        // Error message depends a little on node version.
+        await assert.isRejected(fetch(`${server.serverUrl}/status`), /(request.*failed)|(ECONNREFUSED)/);
 
       } finally {
         await server.stop();