(core) Endpoint to report on the latest version of stable grist-core image

Summary:
New endpoint `/api/version` that returns latest version of stable docker image in format:
```
{"latestVersion":"1.1.12","
  updatedAt":"2024-03-06T06:28:25.752337Z","
  isCritical":false,
 "updateURL":"https://hub.docker.com/r/gristlabs/grist"
}
```

It connects to docker hub API and reads the version from the tag lists endpoint.
Stores telemetry passed from the client such us: current version, deployment type, installationId and others.

Test Plan: Added new test

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D4220

app/common/SortFunc.ts

@@ -28,7 +28,7 @@ type Comparator = (val1: any, val2: any) => number;
  * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/localeCompare
  */
 const collator = new Intl.Collator(undefined, {numeric: true});
-function naturalCompare(val1: any, val2: any) {
+export function naturalCompare(val1: any, val2: any) {
   if (typeof val1 === 'string' && typeof val2 === 'string') {
     return collator.compare(val1, val2);
   }

app/common/Telemetry.ts

@@ -1740,6 +1740,22 @@ export const TelemetryContracts: TelemetryContracts = {
       },
     },
   },
+  checkedUpdateAPI: {
+    category: "SelfHosted",
+    description: 'Triggered when the app checks for updates.',
+    minimumTelemetryLevel: Level.limited,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      installationId: {
+        description: 'The installation id of the client.',
+        dataType: 'string',
+      },
+      deploymentType: {
+        description: 'The deployment type of the client.',
+        dataType: 'string',
+      },
+    },
+  }
 };
 
 type TelemetryContracts = Record<TelemetryEvent, TelemetryEventContract>;
@@ -1810,6 +1826,7 @@ export const TelemetryEvents = StringUnion(
   'visitedForm',
   'submittedForm',
   'changedAccessRules',
+  'checkedUpdateAPI'
 );
 export type TelemetryEvent = typeof TelemetryEvents.type;
 
@@ -1824,7 +1841,8 @@ type TelemetryEventCategory =
   | 'TeamSite'
   | 'ProductVisits'
   | 'AccessRules'
-  | 'WidgetUsage';
+  | 'WidgetUsage'
+  | 'SelfHosted';
 
 interface TelemetryEventContract {
   description: string;

app/server/lib/FlexServer.ts

@@ -67,6 +67,7 @@ import {TagChecker} from 'app/server/lib/TagChecker';
 import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
 import {startTestingHooks} from 'app/server/lib/TestingHooks';
 import {getTestLoginSystem} from 'app/server/lib/TestLogin';
+import {UpdateManager} from 'app/server/lib/UpdateManager';
 import {addUploadRoute} from 'app/server/lib/uploads';
 import {buildWidgetRepository, getWidgetsInPlugins, IWidgetRepository} from 'app/server/lib/WidgetRepository';
 import {setupLocale} from 'app/server/localization';
@@ -179,6 +180,7 @@ export class FlexServer implements GristServer {
   // Set once ready() is called
   private _isReady: boolean = false;
   private _probes: BootProbes;
+  private _updateManager: UpdateManager;
 
   constructor(public port: number, public name: string = 'flexServer',
               public readonly options: FlexServerOptions = {}) {
@@ -405,6 +407,11 @@ export class FlexServer implements GristServer {
     return this._accessTokens;
   }
 
+  public getUpdateManager() {
+    if (!this._updateManager) { throw new Error('no UpdateManager available'); }
+    return this._updateManager;
+  }
+
   public sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise<void> {
     if (!this._sendAppPage) { throw new Error('no _sendAppPage method available'); }
     return this._sendAppPage(req, resp, options);
@@ -880,6 +887,7 @@ export class FlexServer implements GristServer {
 
   public async close() {
     this._processMonitorStop?.();
+    await this._updateManager?.clear();
     if (this.usage)  { await this.usage.close(); }
     if (this._hosts) { this._hosts.close(); }
     if (this._dbManager) {
@@ -1863,6 +1871,16 @@ export class FlexServer implements GristServer {
     return process.env.GRIST_TEST_LOGIN ? getTestLoginSystem() : (this._getLoginSystem?.() || getLoginSystem());
   }
 
+  public addUpdatesCheck() {
+    if (this._check('update')) { return; }
+
+    // For now we only are active for sass deployments.
+    if (this._deploymentType !== 'saas') { return; }
+
+    this._updateManager = new UpdateManager(this.app, this);
+    this._updateManager.addEndpoints();
+  }
+
   // Adds endpoints that support imports and exports.
   private _addSupportPaths(docAccessMiddleware: express.RequestHandler[]) {
     if (!this._docWorker) { throw new Error("need DocWorker"); }

app/server/lib/UpdateManager.ts

@@ -0,0 +1,263 @@
+import { ApiError } from "app/common/ApiError";
+import { MapWithTTL } from "app/common/AsyncCreate";
+import { GristDeploymentType } from "app/common/gristUrls";
+import { naturalCompare } from "app/common/SortFunc";
+import { RequestWithLogin } from "app/server/lib/Authorizer";
+import { GristServer } from "app/server/lib/GristServer";
+import { optIntegerParam, optStringParam } from "app/server/lib/requestUtils";
+import { AbortController, AbortSignal } from 'node-abort-controller';
+import type * as express from "express";
+import fetch from "node-fetch";
+import {expressWrap} from 'app/server/lib/expressWrap';
+
+
+// URL to show to the client where the new version for docker based deployments can be found.
+const DOCKER_IMAGE_SITE = "https://hub.docker.com/r/gristlabs/grist";
+
+// URL to show to the client where the new version for docker based deployments can be found.
+const DOCKER_ENDPOINT = process.env.GRIST_TEST_UPDATE_DOCKER_HUB_URL ||
+                          "https://hub.docker.com/v2/namespaces/gristlabs/repositories/grist/tags";
+// Timeout for the request to the external resource.
+const REQUEST_TIMEOUT = optIntegerParam(process.env.GRIST_TEST_UPDATE_REQUEST_TIMEOUT, '') ?? 10000; // 10s
+// Delay between retries in case of rate limiting.
+const RETRY_TIMEOUT = optIntegerParam(process.env.GRIST_TEST_UPDATE_RETRY_TIMEOUT, '') ?? 4000; // 4s
+// We cache the good result for an hour.
+const GOOD_RESULT_TTL = optIntegerParam(process.env.GRIST_TEST_UPDATE_CHECK_TTL, '') ?? 60 * 60 * 1000; // 1h
+// We cache the bad result errors from external resources for a minute.
+const BAD_RESULT_TTL = optIntegerParam(process.env.GRIST_TEST_UPDATE_ERROR_TTL, '') ?? 60 * 1000; // 1m
+
+// A hook for tests to override the default values.
+export const Deps = {
+  DOCKER_IMAGE_SITE,
+  DOCKER_ENDPOINT,
+  REQUEST_TIMEOUT,
+  RETRY_TIMEOUT,
+  GOOD_RESULT_TTL,
+  BAD_RESULT_TTL,
+};
+
+export class UpdateManager {
+
+  // Cache for the latest version of the client.
+  private _latestVersion: MapWithTTL<
+    GristDeploymentType,
+    // We cache the promise, so that we can wait for the first request.
+    // This promise will always resolves, but can be resolved with an error.
+    Promise<ApiError|LatestVersion>
+  >;
+
+  private _abortController = new AbortController();
+
+  public constructor(
+    private _app: express.Application,
+    private _server: GristServer
+  ) {
+    this._latestVersion = new MapWithTTL<GristDeploymentType, Promise<ApiError|LatestVersion>>(Deps.GOOD_RESULT_TTL);
+  }
+
+  public addEndpoints() {
+    // Make sure that config is ok, so that we are not surprised when client asks as about that.
+    if (Deps.DOCKER_ENDPOINT) {
+      try {
+        new URL(Deps.DOCKER_ENDPOINT);
+      } catch (err) {
+        throw new Error(
+          `Invalid value for GRIST_UPDATE_DOCKER_URL, expected URL: ${Deps.DOCKER_ENDPOINT}`
+        );
+      }
+    }
+
+    // Support both POST and GET requests.
+    this._app.use("/api/version", expressWrap(async (req, res) => {
+      // Get some telemetry from the body request.
+      const payload = (name: string) => req.body?.[name] ?? req.query[name];
+
+      // This is the most interesting part for us, to track installation ids and match them
+      // with the version of the client. Won't be send without telemetry opt in.
+      const installationId = optStringParam(
+        payload("installationId"),
+        "installationId"
+      );
+
+      // Current version of grist-core part of the client. Currently not used and not
+      // passed from the client.
+
+      // Deployment type of the client (we expect this to be 'core' for most of the cases).
+      const deploymentType = optStringParam(
+        payload("deploymentType"),
+        "deploymentType"
+      ) as GristDeploymentType|undefined;
+
+      this._server
+        .getTelemetry()
+        .logEvent(req as RequestWithLogin, "checkedUpdateAPI", {
+          full: {
+            installationId,
+            deploymentType,
+          },
+        });
+
+      // For now we will just check the latest tag of docker stable image, assuming
+      // that this is what the client wants. In the future we might have different
+      // implementation based on the client deployment type.
+      const deploymentToCheck = 'core';
+      const versionChecker: VersionChecker = getLatestStableDockerVersion;
+
+      // To not spam the docker hub with requests, we will cache the good result for an hour.
+      // We are actually caching the promise, so subsequent requests will wait for the first one.
+      if (!this._latestVersion.has(deploymentToCheck)) {
+        const task = versionChecker(this._abortController.signal).catch(err => err);
+        this._latestVersion.set(deploymentToCheck, task);
+      }
+      const resData = await this._latestVersion.get(deploymentToCheck)!;
+      if (resData instanceof ApiError) {
+        // If the request has failed for any reason, we will throw the error to the client,
+        // but shorten the TTL to 1 minute, so that the next client will try after that time.
+        this._latestVersion.setWithCustomTTL(deploymentToCheck, Promise.resolve(resData), Deps.BAD_RESULT_TTL);
+        throw resData;
+      }
+      res.json(resData);
+    }));
+  }
+
+  public async clear() {
+    this._abortController.abort();
+    for (const task of this._latestVersion.values()) {
+      await task.catch(() => {});
+    }
+    this._latestVersion.clear();
+
+    // This function just clears cache and state, we should end with a fine state.
+    this._abortController = new AbortController();
+  }
+}
+
+/**
+ * JSON returned to the client (exported for tests).
+ */
+export interface LatestVersion {
+  /**
+   * Latest version of core component of the client.
+   */
+  latestVersion: string;
+  /**
+   * If there were any critical updates after client's version. Undefined if
+   * we don't know client version or couldn't figure this out for some other reason.
+   */
+  isCritical?: boolean;
+  /**
+   * Url where the client can download the latest version (if applicable)
+   */
+  updateURL?: string;
+
+  /**
+   * When the latest version was updated (in ISO format).
+   */
+  updatedAt?: string;
+}
+
+type VersionChecker = (signal: AbortSignal) => Promise<LatestVersion>;
+
+/**
+ * Get the latest stable version of docker image from the hub.
+ */
+export async function getLatestStableDockerVersion(signal: AbortSignal): Promise<LatestVersion> {
+  try {
+    // Find stable tag.
+    const tags = await listRepositoryTags(signal);
+    const stableTag = tags.find((tag) => tag.name === "stable");
+    if (!stableTag) {
+      throw new ApiError("No stable tag found", 404);
+    }
+
+    // Now find all tags with the same image.
+    const up = tags
+      // Filter by digest.
+      .filter((tag) => tag.digest === stableTag.digest)
+      // Name should be a version number in a correct format (should start with a number or v and number).
+      .filter(tag => /^v?\d+/.test(tag.name))
+      // And sort it in natural order (so that 1.1.10 is after 1.1.9).
+      .sort(compare("name"));
+
+    const last = up[up.length - 1];
+    // Panic if we don't have any tags that looks like version numbers.
+    if (!last) {
+      throw new ApiError("No stable image found", 404);
+    }
+    return {
+      latestVersion: last.name,
+      updatedAt: last.tag_last_pushed,
+      isCritical: false,
+      updateURL: Deps.DOCKER_IMAGE_SITE
+    };
+  } catch (err) {
+    // Make sure to throw only ApiErrors (cache depends on that).
+    if (err instanceof ApiError) {
+      throw err;
+    }
+    throw new ApiError(err.message, 500);
+  }
+}
+
+// Shape of the data from the Docker Hub API.
+interface DockerTag {
+  name: string;
+  digest: string;
+  tag_last_pushed: string;
+}
+
+interface DockerResponse {
+  results: DockerTag[];
+  next: string|null;
+}
+
+// https://docs.docker.com/docker-hub/api/latest/#tag/repositories/
+// paths/~1v2~1namespaces~1%7Bnamespace%7D~1repositories~1%7Brepository%7D~1tags/get
+async function listRepositoryTags(signal: AbortSignal): Promise<DockerTag[]>{
+  const tags: DockerTag[] = [];
+
+  // In case of rate limiting, we will retry the request 20 times.
+  // This is for all pages, so we might hit the limit multiple times.
+  let MAX_RETRIES = 20;
+
+  const url = new URL(Deps.DOCKER_ENDPOINT);
+  url.searchParams.set("page_size", "100");
+  let next: string|null = url.toString();
+
+  // We assume have a maximum of 100 000 tags, if that is not enough, we will have to change this.
+  let MAX_LOOPS = 1000;
+
+  while (next && MAX_LOOPS-- > 0) {
+    const response = await fetch(next, {signal, timeout: Deps.REQUEST_TIMEOUT});
+    if (response.status === 429) {
+      // We hit the rate limit, let's wait a bit and try again.
+      await new Promise((resolve) => setTimeout(resolve, Deps.RETRY_TIMEOUT));
+      if (signal.aborted) {
+        throw new Error("Aborted");
+      }
+      if (MAX_RETRIES-- <= 0) {
+        throw new Error("Too many retries");
+      }
+      continue;
+    }
+    if (response.status !== 200) {
+      throw new ApiError(await response.text(), response.status);
+    }
+    const json: DockerResponse = await response.json();
+    tags.push(...json.results);
+    next = json.next;
+  }
+  if (MAX_LOOPS <= 0) {
+    throw new Error("Too many tags found");
+  }
+  return tags;
+}
+
+/**
+ * Helper for sorting in natural order (1.1.10 is after 1.1.9).
+ */
+function compare<T>(prop: keyof T) {
+  return (a: T, b: T) => {
+    return naturalCompare(a[prop], b[prop]);
+  };
+}

app/server/mergedServerMain.ts

@@ -106,6 +106,7 @@ export async function main(port: number, serverTypes: ServerType[],
   server.addHealthCheck();
   if (includeHome || includeApp) {
     server.addBootPage();
+    server.addUpdatesCheck();
   }
   server.denyRequestsIfNotReady();
 

test/gen-server/UpdateChecks.ts

@@ -0,0 +1,347 @@
+import axios from "axios";
+import * as chai from "chai";
+import * as sinon from 'sinon';
+
+import { configForUser } from "test/gen-server/testUtils";
+import * as testUtils from "test/server/testUtils";
+import { serveSomething, Serving } from "test/server/customUtil";
+import { Deps, LatestVersion } from "app/server/lib/UpdateManager";
+import { TestServer } from "test/gen-server/apiUtils";
+import { delay } from "app/common/delay";
+
+const assert = chai.assert;
+
+let testServer: TestServer;
+
+const stop = async () => {
+  await testServer?.stop();
+  testServer = null as any;
+};
+
+let homeUrl: string;
+let dockerHub: Serving & { signal: () => Defer };
+
+const chimpy = configForUser("Chimpy");
+
+// Tests specific complex scenarios that may have previously resulted in wrong behavior.
+describe("UpdateChecks", function () {
+  testUtils.setTmpLogLevel("error");
+
+  this.timeout("20s");
+
+  const sandbox = sinon.createSandbox();
+
+  before(async function () {
+    testUtils.EnvironmentSnapshot.push();
+    dockerHub = await dummyDockerHub();
+    assert.equal((await fetch(dockerHub.url + "/tags")).status, 200);
+
+    // Start the server with correct configuration.
+    Object.assign(process.env, {
+      GRIST_TEST_SERVER_DEPLOYMENT_TYPE: "saas",
+    });
+    sandbox.stub(Deps, "REQUEST_TIMEOUT").value(300);
+    sandbox.stub(Deps, "RETRY_TIMEOUT").value(400);
+    sandbox.stub(Deps, "GOOD_RESULT_TTL").value(500);
+    sandbox.stub(Deps, "BAD_RESULT_TTL").value(200);
+    sandbox.stub(Deps, "DOCKER_ENDPOINT").value(dockerHub.url + "/tags");
+
+    await startInProcess(this);
+  });
+
+  after(async function () {
+    sandbox.restore();
+    await dockerHub.shutdown();
+    await stop();
+    testUtils.EnvironmentSnapshot.pop();
+  });
+
+  afterEach(async function () {
+    await testServer.server.getUpdateManager().clear();
+  });
+
+  it("should read latest version as anonymous user in happy path", async function () {
+    setEndpoint(dockerHub.url + "/tags");
+    const resp = await axios.get(`${homeUrl}/api/version`);
+    assert.equal(resp.status, 200, `${homeUrl}/api/version`);
+    const result: LatestVersion = resp.data;
+    assert.equal(result.latestVersion, "10");
+
+    // Also works in post method.
+    const resp2 = await axios.post(`${homeUrl}/api/version`);
+    assert.equal(resp2.status, 200);
+    assert.deepEqual(resp2.data, result);
+  });
+
+  it("should read latest version as existing user", async function () {
+    setEndpoint(dockerHub.url + "/tags");
+    const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(resp.status, 200);
+    const result: LatestVersion = resp.data;
+    assert.equal(result.latestVersion, "10");
+  });
+
+  it("passes errors to client", async function () {
+    setEndpoint(dockerHub.url + "/404");
+    const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(resp.status, 404);
+    assert.deepEqual(resp.data, { error: "Not Found" });
+  });
+
+  it("retries on 429", async function () {
+    setEndpoint(dockerHub.url + "/429");
+
+    // First make sure that mock works.
+    assert.equal((await fetch(dockerHub.url + "/429")).status, 200);
+    assert.equal((await fetch(dockerHub.url + "/429")).status, 429);
+    assert.equal((await fetch(dockerHub.url + "/429")).status, 200);
+    assert.equal((await fetch(dockerHub.url + "/429")).status, 429);
+
+    // Now make sure that 4 subsequent requests are successful.
+    const check = async () => {
+      const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
+      assert.equal(resp.status, 200);
+      const result: LatestVersion = resp.data;
+      assert.equal(result.latestVersion, "10");
+    };
+
+    await check();
+    await check();
+    await check();
+    await check();
+  });
+
+  it("throws when receives html", async function () {
+    setEndpoint(dockerHub.url + "/html");
+    const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(resp.status, 500);
+  });
+
+  it("caches data end errors", async function () {
+    setEndpoint(dockerHub.url + "/error");
+    const r1 = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(r1.status, 500);
+    assert.equal(r1.data.error, "1");
+
+    const r2 = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(r2.status, 500);
+    assert.equal(r2.data.error, "1"); // since errors are cached for 200ms.
+
+    await delay(300); // error is cached for 200ms
+
+    const r3 = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(r3.status, 500);
+    assert.equal(r3.data.error, "2"); // second error is different, but still cached for 200ms.
+
+    const r4 = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(r4.status, 500);
+    assert.equal(r4.data.error, "2");
+
+    await delay(300);
+
+    // Now we should get correct result, but it will be cached for 500ms.
+
+    const r5 = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(r5.status, 200);
+    assert.equal(r5.data.latestVersion, "3"); // first successful response is cached for 2 seconds.
+
+    const r6 = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(r6.status, 200);
+    assert.equal(r6.data.latestVersion, "3");
+
+    await delay(700);
+
+    const r7 = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(r7.status, 200);
+    assert.equal(r7.data.latestVersion, "4");
+  });
+
+  it("can stop server when hangs", async function () {
+    setEndpoint(dockerHub.url + "/hang");
+    const handCalled = dockerHub.signal();
+    const resp = axios
+      .get(`${homeUrl}/api/version`, chimpy)
+      .catch((err) => ({ status: 999, data: null }));
+    await handCalled;
+    await stop();
+    const result = await resp;
+    assert.equal(result.status, 500);
+    assert.match(result.data.error, /aborted/);
+    // Start server again, and make sure it works.
+    await startInProcess(this);
+  });
+
+  it("dosent starts for non saas deployment", async function () {
+    try {
+      testUtils.EnvironmentSnapshot.push();
+      Object.assign(process.env, {
+        GRIST_TEST_SERVER_DEPLOYMENT_TYPE: "core",
+      });
+      await stop();
+      await startInProcess(this);
+      const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
+      assert.equal(resp.status, 404);
+    } finally {
+      testUtils.EnvironmentSnapshot.pop();
+    }
+
+    // Start normal one again.
+    await stop();
+    await startInProcess(this);
+  });
+
+  it("reports error when timeout happens", async function () {
+    setEndpoint(dockerHub.url + "/timeout");
+    const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
+    assert.equal(resp.status, 500);
+    assert.match(resp.data.error, /timeout/);
+  });
+});
+
+async function dummyDockerHub() {
+  let odds = 0;
+
+  // We offer a way to signal when request is received.
+  // Test can add a dummy promise using signal() method, and it is resolved
+  // when any request is received.
+  const signals: Defer[] = [];
+  let errorCount = 0;
+
+  const tempServer = await serveSomething((app) => {
+    app.use((req, res, next) => {
+      signals.forEach((p) => p.resolve());
+      signals.length = 0;
+      next();
+    });
+    app.get("/404", (_, res) => res.status(404).send("Not Found").end());
+    app.get("/429", (_, res) => {
+      if (odds++ % 2) {
+        res.status(429).send("Too Many Requests");
+      } else {
+        res.json(SECOND_PAGE);
+      }
+    });
+    app.get("/timeout", (_, res) => {
+      setTimeout(() => res.status(200).json(SECOND_PAGE), 500);
+    });
+
+    app.get("/error", (_, res) => {
+      errorCount++;
+      // First 2 calls will return error, next will return numbers (3, 4, 5, 6, 7, 8, 9, 10)
+      if (errorCount <= 2) {
+        res.status(500).send(String(errorCount));
+      } else {
+        res.json(VERSION(errorCount));
+      }
+    });
+
+    app.get("/html", (_, res) => {
+      res.status(200).send("<html></html>");
+    });
+    app.get("/hang", () => {});
+    app.get("/tags", (_, res) => {
+      res.status(200).json(FIRST_PAGE(tempServer));
+    });
+    app.get("/next", (_, res) => {
+      res.status(200).json(SECOND_PAGE);
+    });
+  });
+
+  return Object.assign(tempServer, {
+    signal() {
+      const p = defer();
+      signals.push(p);
+      return p;
+    },
+  });
+}
+
+function setEndpoint(endpoint: string) {
+  sinon.stub(Deps, "DOCKER_ENDPOINT").value(endpoint);
+}
+
+async function startInProcess(context: Mocha.Context) {
+  testServer = new TestServer(context);
+  await testServer.start(["home"]);
+  homeUrl = testServer.serverUrl;
+}
+
+const VERSION = (i: number) => ({
+  results: [
+    {
+      tag_last_pushed: "2024-03-26T07:11:01.272113Z",
+      name: "stable",
+      digest: "stable",
+    },
+    {
+      tag_last_pushed: "2024-03-26T07:11:01.272113Z",
+      name: i.toString(),
+      digest: "stable",
+    },
+  ],
+  count: 2,
+  next: null,
+});
+
+const SECOND_PAGE = {
+  results: [
+    {
+      tag_last_pushed: "2024-03-26T07:11:01.272113Z",
+      name: "stable",
+      digest: "stable",
+    },
+    {
+      tag_last_pushed: "2024-03-26T07:11:01.272113Z",
+      name: "latest",
+      digest: "latest",
+    },
+    {
+      tag_last_pushed: "2024-03-26T07:11:01.272113Z",
+      name: "1",
+      digest: "latest",
+    },
+    {
+      tag_last_pushed: "2024-03-26T07:11:01.272113Z",
+      name: "1",
+      digest: "stable",
+    },
+    {
+      tag_last_pushed: "2024-03-26T07:11:01.272113Z",
+      name: "9",
+      digest: "stable",
+    },
+    {
+      tag_last_pushed: "2024-03-26T07:11:01.272113Z",
+      name: "10",
+      digest: "stable",
+    },
+  ],
+  count: 6,
+  next: null,
+};
+
+const FIRST_PAGE = (tempServer: Serving) => ({
+  results: [],
+  count: 0,
+  next: tempServer.url + "/next",
+});
+
+interface Defer {
+  then: Promise<void>["then"];
+  resolve: () => void;
+  reject: () => void;
+}
+
+const defer = () => {
+  let resolve: () => void;
+  let reject: () => void;
+  const promise = new Promise<void>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  }).catch(() => {});
+  return {
+    then: promise.then.bind(promise),
+    resolve: resolve!,
+    reject: reject!,
+  };
+};

test/server/testUtils.ts

@@ -302,11 +302,26 @@ export async function readFixtureDoc(docName: string) {
 // a class to store a snapshot of environment variables, can be reverted to by
 // calling .restore()
 export class EnvironmentSnapshot {
+
+  public static push() {
+    this._stack.push(new EnvironmentSnapshot());
+  }
+
+  public static pop() {
+    const snapshot = this._stack.pop();
+    if (!snapshot) {
+      throw new Error("EnvironmentSnapshot stack is empty");
+    }
+    snapshot.restore();
+  }
+
+  private static _stack: EnvironmentSnapshot[] = [];
+
   private _oldEnv: NodeJS.ProcessEnv;
+
   public constructor() {
     this._oldEnv = clone(process.env);
   }
-
   // Reset environment variables.
   public restore() {
     Object.assign(process.env, this._oldEnv);