From ffa81a78b1c3355328ef5356d098015272cf705c Mon Sep 17 00:00:00 2001 From: Spoffy Date: Sat, 15 Jun 2024 04:52:04 +0100 Subject: [PATCH] Initial attempt --- package.json | 7 +- playwright.config.ts | 81 ++++++++++ test/nbrowser/Boot.playwright.ts | 64 ++++++++ test/nbrowser/gristUtils.ts | 38 +++++ test/nbrowser/playwrightGristUtils.ts | 6 + test/nbrowser/playwrightTestUtils.ts | 205 ++++++++++++++++++++++++++ yarn.lock | 26 ++++ 7 files changed, 425 insertions(+), 2 deletions(-) create mode 100644 playwright.config.ts create mode 100644 test/nbrowser/Boot.playwright.ts create mode 100644 test/nbrowser/playwrightGristUtils.ts create mode 100644 test/nbrowser/playwrightTestUtils.ts diff --git a/package.json b/package.json index 708933bb..b2ed4768 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "install:python3": "buildtools/prepare_python3.sh", "build:prod": "buildtools/build.sh", "start:prod": "sandbox/run.sh", - "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} --slow 8000 $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/nbrowser_with_stubs/**/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'", + "test-all-old": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} --slow 8000 $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/nbrowser_with_stubs/**/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'", "test:nbrowser": "TEST_SUITE=nbrowser TEST_SUITE_FOR_TIMINGS=nbrowser TIMINGS_FILE=test/timings/nbrowser.txt GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser/**/*.js'", "test:stubs": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser_with_stubs/**/*.js'", "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'", @@ -28,7 +28,9 @@ "lint:fix": "eslint --cache --cache-strategy=content --fix .", "lint:ci": "eslint --max-warnings=0 .", "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js", - "generate:schema:ts": "buildtools/update_schema.sh" + "generate:schema:ts": "buildtools/update_schema.sh", + "pretest": "buildtools/build.sh", + "test": "NODE_PATH=_build:_build/stubs yarn playwright test" }, "keywords": [ "grist", @@ -43,6 +45,7 @@ "devDependencies": { "@babel/core": "7.18.5", "@babel/eslint-parser": "7.18.2", + "@playwright/test": "^1.44.1", "@types/accept-language-parser": "1.5.2", "@types/backbone": "1.3.43", "@types/chai": "4.1.7", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 00000000..d66eace6 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,81 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + build: { + external: ['app/**', 'stubs/**', 'test/**', 'static/**'], + }, + testDir: './_build', + testMatch: /.*.playwright.js/, + /* Run tests in files in parallel */ + fullyParallel: false, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Avoid being parallel, tests might end up a weird state */ + workers: 1, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://127.0.0.1:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, +/* + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, + */ + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/test/nbrowser/Boot.playwright.ts b/test/nbrowser/Boot.playwright.ts new file mode 100644 index 00000000..ae421920 --- /dev/null +++ b/test/nbrowser/Boot.playwright.ts @@ -0,0 +1,64 @@ +import {assert, driver} from 'mocha-webdriver'; +import * as gu from 'test/nbrowser/playwrightGristUtils'; +import {server, setupTestSuite} from 'test/nbrowser/playwrightTestUtils'; +import * as testUtils from 'test/server/testUtils'; +import { test, expect, Page } from '@playwright/test'; + +/** + * The boot page functionality has been merged with the Admin Panel. + * Check that it behaves as a boot page did now. + */ +test.describe('Boot', () => { + setupTestSuite(); + + let oldEnv: testUtils.EnvironmentSnapshot; + + test.afterEach(({ page }) => gu.checkForErrors(page)); + + async function hasPrompt(page: Page) { + // There is some glitchiness to when the text appears. + const text = await page.getByText(/GRIST_BOOT_KEY/).textContent(); + expect(text).toContain('GRIST_BOOT_KEY=example-'); + } + + test('tells user about /admin', async function() { + await driver.get(`${server.getHost()}/boot`); + assert.match(await driver.getPageSource(), /\/admin/); + // Switch to a regular place to that gu.checkForErrors won't panic - + // it needs a Grist page. + await driver.get(`${server.getHost()}`); + }); + + test('gives prompt about how to enable boot page', async function({ page }) { + await driver.get(`${server.getHost()}/admin`); + await hasPrompt(page); + }); + + test.describe('with a GRIST_BOOT_KEY', function() { + test.beforeAll(async function() { + oldEnv = new testUtils.EnvironmentSnapshot(); + process.env.GRIST_BOOT_KEY = 'lala'; + await server.restart(); + }); + + test.afterAll(async function() { + oldEnv.restore(); + await server.restart(); + }); + + test('gives prompt when key is missing', async function({ page }) { + await driver.get(`${server.getHost()}/admin`); + await hasPrompt(page); + }); + + test('gives prompt when key is wrong', async function({ page }) { + await driver.get(`${server.getHost()}/admin?boot-key=bilbo`); + await hasPrompt(page); + }); + + test('gives page when key is right', async function() { + await driver.get(`${server.getHost()}/admin?boot-key=lala`); + await driver.findContentWait('div', /Is home page available/, 2000); + }); + }); +}); diff --git a/test/nbrowser/gristUtils.ts b/test/nbrowser/gristUtils.ts index 7eea6f44..151f1174 100644 --- a/test/nbrowser/gristUtils.ts +++ b/test/nbrowser/gristUtils.ts @@ -37,6 +37,7 @@ import * as testUtils from 'test/server/testUtils'; import type { AssertionError } from 'assert'; import axios from 'axios'; import { lock } from 'proper-lockfile'; +import { test } from '@playwright/test'; // tslint:disable:no-namespace // Wrap in a namespace so that we can apply stackWrapOwnMethods to all the exports together. @@ -1989,6 +1990,33 @@ export function shareSupportWorkspaceForSuite() { }); } +export function shareSupportWorkspaceForSuitePlaywright() { + let api: UserAPIImpl|undefined; + let wss: Workspace[]|undefined; + + test.beforeAll(async function() { + // test/gen-server/seed.ts creates a support user with a personal org and an "Examples & + // Templates" workspace, but doesn't share it (to avoid impacting the many existing tests). + // Share that workspace with @everyone and @anon, and clean up after this suite. + await addSupportUserIfPossible(); + api = createHomeApi('Support', 'docs'); // this uses an api key, so no need to log in. + wss = await api.getOrgWorkspaces('current'); + await api.updateWorkspacePermissions(wss[0].id, {users: { + 'everyone@getgrist.com': 'viewers', + 'anon@getgrist.com': 'viewers', + }}); + }); + + test.afterAll(async function() { + if (api && wss) { + await api.updateWorkspacePermissions(wss[0].id, {users: { + 'everyone@getgrist.com': null, + 'anon@getgrist.com': null, + }}); + } + }); +} + export async function clearTestState() { await driver.executeScript("window.testGrist = {}"); } @@ -2685,6 +2713,16 @@ export function addSamplesForSuite() { }); } +export function addSamplesForSuitePlaywright() { + test.beforeAll(async function() { + await addSamples(); + }); + + test.afterAll(async function() { + await removeTemplatesOrg(); + }); +} + export async function openAccountMenu() { await driver.findWait('.test-dm-account', 1000).click(); // Since the AccountWidget loads orgs and the user data asynchronously, the menu diff --git a/test/nbrowser/playwrightGristUtils.ts b/test/nbrowser/playwrightGristUtils.ts new file mode 100644 index 00000000..275d9978 --- /dev/null +++ b/test/nbrowser/playwrightGristUtils.ts @@ -0,0 +1,6 @@ +import { expect, Page } from '@playwright/test'; + +export async function checkForErrors(page: Page) { + const errors = await page.evaluate(() => (window as any).getAppErrors()); + expect(errors).toEqual([]); +} diff --git a/test/nbrowser/playwrightTestUtils.ts b/test/nbrowser/playwrightTestUtils.ts new file mode 100644 index 00000000..8b0c0862 --- /dev/null +++ b/test/nbrowser/playwrightTestUtils.ts @@ -0,0 +1,205 @@ +/** + * Exports `server`, set up to start using setupTestSuite(), e.g. + * + * import {assert, driver} from 'mocha-webdriver'; + * import {server, setupTestSuite} from 'test/nbrowser/testUtils'; + * + * describe("MyTest", function() { + * this.timeout(20000); // Needed because we wait for server for up to 15s. + * setupTestSuite(); + * }); + * + * Run with VERBOSE=1 in the environment to see the server log on the console. Normally it goes + * into a file whose path is printed when server starts. + * + * Run `bin/mocha 'test/nbrowser/*.ts' -b --no-exit` to open a command-line prompt on + * first-failure for debugging and quick reruns. + */ +import * as gu from 'test/nbrowser/gristUtils'; +import { server } from 'test/nbrowser/testServer'; +import { test } from '@playwright/test'; + +// Exports the server object with useful methods such as getHost(), waitServerReady(), +// simulateLogin(), etc. +export {server}; + +interface TestSuiteOptions { + samples?: boolean; + team?: boolean; + + // If set, clear user preferences for all test users at the end of the suite. It should be used + // for suites that modify preferences. Not that it only works in dev, not in deployment tests. + clearUserPrefs?: boolean; + + // Max milliseconds to wait for a page to finish loading. E.g. affects clicks that cause + // navigation, which wait for that. A navigation that takes longer will throw an exception. + pageLoadTimeout?: number; +} + +// Sets up the test suite to use the Grist server, and also to record logs and screenshots after +// failed tests (if MOCHA_WEBDRIVER_LOGDIR var is set). +// +// Returns a Cleanup instance as a convenience, for use scheduling any clean-up that would have +// the same scope as the test suite. +export function setupTestSuite(options?: TestSuiteOptions) { + test.beforeAll(async () => server.start()); + test.afterAll(async () => server.stop()); + + // After every suite, assert it didn't leave new browser windows open. + // Don't know if we need this in playwright + //checkForExtraWindows(); + + // After every suite, clear sessionStorage and localStorage to avoid affecting other tests. + if (!process.env.NO_CLEANUP) { + // Not sure this works in playwright? or is needed? + //test.afterAll(({ page }) => clearCurrentWindowStorage(page)); + } + // Also, log out, to avoid logins interacting, unless NO_CLEANUP is requested (useful for + // debugging tests). + if (!process.env.NO_CLEANUP) { + test.afterAll(() => server.removeLogin()); + } + + // If requested, clear user preferences for all test users after this suite. + if (options?.clearUserPrefs) { + test.afterAll(clearTestUserPreferences); + } + + // Though unlikely it is possible that the server was left paused by a previous test, so let's + // always call resume. + test.afterEach(() => server.resume()); + + // Close database until next test explicitly needs it, to avoid conflicts + // with tests that don't use the same server. + test.afterAll(async () => server.closeDatabase()); + + // Not needed in playwright? Uses different timing model. + /* + if (options?.pageLoadTimeout) { + setDriverTimeoutsForSuite({pageLoad: options.pageLoadTimeout}); + } + */ + + return setupRequirement({team: true, ...options}); +} + +async function clearTestUserPreferences() { + // After every suite, clear user preferences for all test users. + const dbManager = await server.getDatabase(); + let emails = Object.keys(gu.TestUserEnum).map(user => gu.translateUser(user as any).email); + emails = [...new Set(emails)]; // Remove duplicates. + await dbManager.testClearUserPrefs(emails); +} + +export type CleanupFunc = (() => void|Promise); + +/** + * Helper to run cleanup callbacks created in a test case. See setupCleanup() below for usage. + */ +export class Cleanup { + private _callbacksAfterAll: CleanupFunc[] = []; + private _callbacksAfterEach: CleanupFunc[] = []; + + public addAfterAll(cleanupFunc: CleanupFunc) { + this._callbacksAfterAll.push(cleanupFunc); + } + public addAfterEach(cleanupFunc: CleanupFunc) { + this._callbacksAfterEach.push(cleanupFunc); + } + + public async runCleanup(which: 'all'|'each') { + const callbacks = which === 'all' ? this._callbacksAfterAll : this._callbacksAfterEach; + const list = callbacks.splice(0); // Get a copy of the list AND clear it out. + for (const f of list) { + await f(); + } + } +} + +/** + * Helper to run cleanup callbacks created in the course of running a test. + * Usage: + * const cleanup = setupCleanup(); + * it("should do stuff", function() { + * cleanup.addAfterAll(() => { ...doSomething1()... }); + * cleanup.addAfterEach(() => { ...doSomething2()... }); + * }); + * + * Here, doSomething1() is called at the end of a suite, while doSomething2() is called at the end + * of the current test case. + */ +export function setupCleanup() { + const cleanup = new Cleanup(); + test.afterAll(() => cleanup.runCleanup('all')); + test.afterEach(() => cleanup.runCleanup('each')); + return cleanup; +} + +/** + * Implement some optional requirements for a test, such as having an example document + * present, or a team site to run tests in. These requirements should be automatically + * satisfied by staging/prod deployments, and only need doing in self-contained tests + * or tests against dev servers. + * + * Returns a Cleanup instance for any cleanup that would have the same scope as the + * requirement. + */ +export function setupRequirement(options: TestSuiteOptions) { + const cleanup = setupCleanup(); + if (options.samples) { + if (process.env.TEST_ADD_SAMPLES || !server.isExternalServer()) { + gu.shareSupportWorkspaceForSuitePlaywright(); // TODO: Remove after the support workspace is removed from the backend. + gu.addSamplesForSuitePlaywright(); + } + } + + before(async function() { + + if (new URL(server.getHost()).hostname !== 'localhost') { + // Non-dev servers should already meet the requirements; in any case we should not + // fiddle with them here. + return; + } + + // Optionally ensure that a team site is available for tests. + if (options.team) { + await gu.addSupportUserIfPossible(); + const api = gu.createHomeApi('support', 'docs'); + for (const suffix of ['', '2'] as const) { + let orgName = `test${suffix}-grist`; + const deployment = process.env.GRIST_ID_PREFIX; + if (deployment) { orgName = `${orgName}-${deployment}`; } + let isNew: boolean = false; + try { + await api.newOrg({name: `Test${suffix} Grist`, domain: orgName}); + isNew = true; + } catch (e) { + // Assume the org already exists. + } + if (isNew) { + await api.updateOrgPermissions(orgName, { + users: { + 'gristoid+chimpy@gmail.com': 'owners', + } + }); + // Recreate the api for the correct org, then update billing. + const api2 = gu.createHomeApi('support', orgName); + const billing = api2.getBillingAPI(); + try { + await billing.updateBillingManagers({ + users: { + 'gristoid+chimpy@gmail.com': 'managers', + } + }); + } catch (e) { + // ignore if no billing endpoint + if (!String(e).match('404: Not Found')) { + throw e; + } + } + } + } + } + }); + return cleanup; +} diff --git a/yarn.lock b/yarn.lock index 798534bf..a14bf312 100644 --- a/yarn.lock +++ b/yarn.lock @@ -639,6 +639,13 @@ "@otplib/plugin-crypto" "^12.0.1" "@otplib/plugin-thirty-two" "^12.0.1" +"@playwright/test@^1.44.1": + version "1.44.1" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.44.1.tgz#cc874ec31342479ad99838040e99b5f604299bcb" + integrity sha512-1hZ4TNvD5z9VuhNJ/walIjvMVvYkZKf71axoF/uiAqpntQJXpG64dlXhoDXE3OczPuTuvjf/M5KWFg5VAVUS3Q== + dependencies: + playwright "1.44.1" + "@popperjs/core@2.3.3": version "2.3.3" resolved "https://registry.npmjs.org/@popperjs/core/-/core-2.3.3.tgz" @@ -4020,6 +4027,11 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== +fsevents@2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== + fsevents@~2.1.2: version "2.1.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e" @@ -6511,6 +6523,20 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +playwright-core@1.44.1: + version "1.44.1" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.44.1.tgz#53ec975503b763af6fc1a7aa995f34bc09ff447c" + integrity sha512-wh0JWtYTrhv1+OSsLPgFzGzt67Y7BE/ZS3jEqgGBlp2ppp1ZDj8c+9IARNW4dwf1poq5MgHreEM2KV/GuR4cFA== + +playwright@1.44.1: + version "1.44.1" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.44.1.tgz#5634369d777111c1eea9180430b7a184028e7892" + integrity sha512-qr/0UJ5CFAtloI3avF95Y0L1xQo6r3LQArLIg/z/PoGJ6xa+EwzrwO5lpNr/09STxdHuUoP2mvuELJS+hLdtgg== + dependencies: + playwright-core "1.44.1" + optionalDependencies: + fsevents "2.3.2" + plotly.js-basic-dist@2.13.2: version "2.13.2" resolved "https://registry.npmjs.org/plotly.js-basic-dist/-/plotly.js-basic-dist-2.13.2.tgz"