(core) Move Billing tests into a separate test suite, and skip some

Summary:
- Also move out reused helper methods out of test suites, to avoid inadvertently including some test suites into other suites.
- Add logging around onNewTab since that seems related to test failures.
- Skip several tests in Billing that fail particularly often.

Test Plan: Checking that tests pass, and how many. Compared to the last successful run on master, there are 4 new skips (this diff), and 3 fewer test cases (a test case run in triplicate was removed in https://github.com/gristlabs/grist-core/pull/899)

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D4216

test/nbrowser/gristUtils.ts

@@ -31,7 +31,8 @@ import { GristWebDriverUtils, PageWidgetPickerOptions,
          WindowDimensions as WindowDimensionsBase } from 'test/nbrowser/gristWebDriverUtils';
 import { HomeUtil } from 'test/nbrowser/homeUtil';
 import { server } from 'test/nbrowser/testServer';
-import { Cleanup } from 'test/nbrowser/testUtils';
+import type { Cleanup } from 'test/nbrowser/testUtils';
+import { fetchScreenshotAndLogs } from 'test/nbrowser/webdriverUtils';
 import * as testUtils from 'test/server/testUtils';
 import type { AssertionError } from 'assert';
 import axios from 'axios';
@@ -1067,6 +1068,14 @@ export async function hideBanners() {
     document.head.appendChild(style);`);
 }
 
+export async function assertBannerText(text: string | null) {
+  if (text === null) {
+    assert.isFalse(await driver.find('.test-banner').isPresent());
+  } else {
+    assert.equal(await driver.findWait('.test-doc-usage-banner-text', 2000).getText(), text);
+  }
+}
+
 /**
  * Returns the left-panel item for the given page, given by a full string name, or a RegExp.
  * You may simply click it to switch to that page.
@@ -2826,23 +2835,34 @@ export async function getEnabledOptions(): Promise<SortOption[]> {
  * Runs action in a separate tab, closing the tab after.
  * In case of an error tab is not closed, consider using cleanupExtraWindows
  * on whole test suite if needed.
+ *
+ * If {test: this.test} is given in options, we will additionally record a screenshot and driver
+ * logs, named using the test name, before opening the new tab, and before and after closing it.
  */
-export async function onNewTab(action: () => Promise<void>) {
+export async function onNewTab(action: () => Promise<void>, options?: {test?: Mocha.Runnable}) {
   const currentTab = await driver.getWindowHandle();
   await driver.executeScript("window.open('about:blank', '_blank')");
   const tabs = await driver.getAllWindowHandles();
   const newTab = tabs[tabs.length - 1];
+  const test = options?.test;
+  if (test) { await fetchScreenshotAndLogs(test); }
   await driver.switchTo().window(newTab);
   try {
     await action();
+  } catch (e) {
+    console.warn("onNewTab cleaning up tab after error", e);
+    throw e;
   } finally {
+    if (test) { await fetchScreenshotAndLogs(test); }
     const newCurrentTab = await driver.getWindowHandle();
     if (newCurrentTab === newTab) {
       await driver.close();
       await driver.switchTo().window(currentTab);
+      console.log("onNewTab returned to original tab");
     } else {
       console.log("onNewTab not cleaning up because is not on expected tab");
     }
+    if (test) { await fetchScreenshotAndLogs(test); }
   }
 }
 

test/nbrowser/testUtils.ts

@@ -20,8 +20,6 @@ import {addToRepl, assert, driver, enableDebugCapture, ITimeouts,
   Key, setOptionsModifyFunc, useServer} from 'mocha-webdriver';
 import * as gu from 'test/nbrowser/gristUtils';
 import {server} from 'test/nbrowser/testServer';
-import * as path from 'path';
-import * as fs from 'fs/promises';
 
 // Exports the server object with useful methods such as getHost(), waitServerReady(),
 // simulateLogin(), etc.
@@ -317,39 +315,3 @@ export function setupRequirement(options: TestSuiteOptions) {
   });
   return cleanup;
 }
-
-export async function withDriverLogging(
-  test: Mocha.Runnable|undefined, periodMs: number, timeoutMs: number,
-  callback: () => Promise<void>
-) {
-  const dir = process.env.MOCHA_WEBDRIVER_LOGDIR!;
-  assert.isOk(dir, "driverLogging: MOCHA_WEBDRIVER_LOGDIR not set");
-  const testName = test?.file ? path.basename(test.file, path.extname(test.file)) : "unnamed";
-  const logPath = path.resolve(dir, `${testName}-driverLogging.log`);
-  await fs.mkdir(dir, {recursive: true});
-
-  let running = false;
-  async function repeat() {
-    if (running) {
-      console.log("driverLogging: skipping because previous repeat still running");
-      return;
-    }
-    running = true;
-    try {
-      await driver.saveScreenshot(`${testName}-driverLoggingScreenshot-{N}.png`);
-      const messages = await driver.fetchLogs('driver');
-      await fs.appendFile(logPath, messages.join("\n") + "\n");
-    } finally {
-      running = false;
-    }
-  }
-
-  const periodic = setInterval(repeat, periodMs);
-  const timeout = setTimeout(() => clearInterval(periodic), timeoutMs);
-  try {
-    return await callback();
-  } finally {
-    clearInterval(periodic);
-    clearTimeout(timeout);
-  }
-}

test/nbrowser/webdriverUtils.ts

@@ -0,0 +1,44 @@
+import log from 'app/server/lib/log';
+import {assert, driver} from 'mocha-webdriver';
+import * as path from 'path';
+import * as fs from 'fs/promises';
+
+
+export async function fetchScreenshotAndLogs(test: Mocha.Runnable|undefined) {
+  const dir = process.env.MOCHA_WEBDRIVER_LOGDIR!;
+  assert.isOk(dir, "driverLogging: MOCHA_WEBDRIVER_LOGDIR not set");
+  const testName = test?.file ? path.basename(test.file, path.extname(test.file)) : "unnamed";
+  const logPath = path.resolve(dir, `${testName}-driverLogging.log`);
+  await fs.mkdir(dir, {recursive: true});
+  await driver.saveScreenshot(`${testName}-driverLoggingScreenshot-{N}.png`);
+  const messages = await driver.fetchLogs('driver');
+  await fs.appendFile(logPath, messages.join("\n") + "\n");
+}
+
+export async function withDriverLogging(
+  test: Mocha.Runnable|undefined, periodMs: number, timeoutMs: number,
+  callback: () => Promise<void>
+) {
+  let running = false;
+  async function repeat() {
+    if (running) {
+      log.warn("driverLogging: skipping because previous repeat still running");
+      return;
+    }
+    running = true;
+    try {
+      await fetchScreenshotAndLogs(test);
+    } finally {
+      running = false;
+    }
+  }
+
+  const periodic = setInterval(repeat, periodMs);
+  const timeout = setTimeout(() => clearInterval(periodic), timeoutMs);
+  try {
+    return await callback();
+  } finally {
+    clearInterval(periodic);
+    clearTimeout(timeout);
+  }
+}