import { ApiError } from "app/common/ApiError"; import { MapWithTTL } from "app/common/AsyncCreate"; import { GristDeploymentType } from "app/common/gristUrls"; import { LatestVersion } from 'app/common/InstallAPI'; import { naturalCompare } from "app/common/SortFunc"; import { RequestWithLogin } from "app/server/lib/Authorizer"; import { expressWrap } from 'app/server/lib/expressWrap'; import { GristServer } from "app/server/lib/GristServer"; import { optIntegerParam, optStringParam } from "app/server/lib/requestUtils"; import { rateLimit } from 'express-rate-limit'; import { AbortController, AbortSignal } from 'node-abort-controller'; import type * as express from "express"; import fetch from "node-fetch"; // URL to show to the client where the new version for docker based deployments can be found. const DOCKER_IMAGE_SITE = "https://hub.docker.com/r/gristlabs/grist"; // URL to show to the client where the new version for docker based deployments can be found. const DOCKER_ENDPOINT = process.env.GRIST_TEST_UPDATE_DOCKER_HUB_URL || "https://hub.docker.com/v2/namespaces/gristlabs/repositories/grist/tags"; // Timeout for the request to the external resource. const REQUEST_TIMEOUT = optIntegerParam(process.env.GRIST_TEST_UPDATE_REQUEST_TIMEOUT, '') ?? 10000; // 10s // Delay between retries in case of rate limiting. const RETRY_TIMEOUT = optIntegerParam(process.env.GRIST_TEST_UPDATE_RETRY_TIMEOUT, '') ?? 4000; // 4s // We cache the good result for an hour. const GOOD_RESULT_TTL = optIntegerParam(process.env.GRIST_TEST_UPDATE_CHECK_TTL, '') ?? 60 * 60 * 1000; // 1h // We cache the bad result errors from external resources for a minute. const BAD_RESULT_TTL = optIntegerParam(process.env.GRIST_TEST_UPDATE_ERROR_TTL, '') ?? 60 * 1000; // 1m // A hook for tests to override the default values. export const Deps = { DOCKER_IMAGE_SITE, DOCKER_ENDPOINT, REQUEST_TIMEOUT, RETRY_TIMEOUT, GOOD_RESULT_TTL, BAD_RESULT_TTL, }; export class UpdateManager { // Cache for the latest version of the client. private _latestVersion: MapWithTTL< GristDeploymentType, // We cache the promise, so that we can wait for the first request. // This promise will always resolves, but can be resolved with an error. Promise >; private _abortController = new AbortController(); public constructor( private _app: express.Application, private _server: GristServer ) { this._latestVersion = new MapWithTTL>(Deps.GOOD_RESULT_TTL); } public addEndpoints() { // Make sure that config is ok, so that we are not surprised when client asks as about that. if (Deps.DOCKER_ENDPOINT) { try { new URL(Deps.DOCKER_ENDPOINT); } catch (err) { throw new Error( `Invalid value for GRIST_UPDATE_DOCKER_URL, expected URL: ${Deps.DOCKER_ENDPOINT}` ); } } // Rate limit the requests to the version API, so that we don't get spammed. // 30 requests per second, per IP. The requests are cached so, we should be fine, but make // sure it doesn't get out of hand. On dev laptop I could go up to 600 requests per second. // (30 was picked by hand, to not hit the limit during tests). const limiter = rateLimit({ windowMs: 1000, limit: 30, legacyHeaders: true, }); // Support both POST and GET requests. this._app.use("/api/version", limiter, expressWrap(async (req, res) => { // Get some telemetry from the body request. const payload = (name: string) => req.body?.[name] ?? req.query[name]; // This is the most interesting part for us, to track installation ids and match them // with the version of the client. Won't be send without telemetry opt in. const installationId = optStringParam( payload("installationId"), "installationId" ); // Current version of grist-core part of the client. Currently not used and not // passed from the client. // Deployment type of the client (we expect this to be 'core' for most of the cases). const deploymentType = optStringParam( payload("deploymentType"), "deploymentType" ) as GristDeploymentType|undefined; this._server .getTelemetry() .logEvent(req as RequestWithLogin, "checkedUpdateAPI", { full: { installationId, deploymentType, }, }); // For now we will just check the latest tag of docker stable image, assuming // that this is what the client wants. In the future we might have different // implementation based on the client deployment type. const deploymentToCheck = 'core'; const versionChecker: VersionChecker = getLatestStableDockerVersion; // To not spam the docker hub with requests, we will cache the good result for an hour. // We are actually caching the promise, so subsequent requests will wait for the first one. if (!this._latestVersion.has(deploymentToCheck)) { const task = versionChecker(this._abortController.signal).catch(err => err); this._latestVersion.set(deploymentToCheck, task); } const resData = await this._latestVersion.get(deploymentToCheck)!; if (resData instanceof ApiError) { // If the request has failed for any reason, we will throw the error to the client, // but shorten the TTL to 1 minute, so that the next client will try after that time. this._latestVersion.setWithCustomTTL(deploymentToCheck, Promise.resolve(resData), Deps.BAD_RESULT_TTL); throw resData; } res.json(resData); })); } public async clear() { this._abortController.abort(); for (const task of this._latestVersion.values()) { await task.catch(() => {}); } this._latestVersion.clear(); // This function just clears cache and state, we should end with a fine state. this._abortController = new AbortController(); } } type VersionChecker = (signal: AbortSignal) => Promise; /** * Get the latest stable version of docker image from the hub. */ export async function getLatestStableDockerVersion(signal: AbortSignal): Promise { try { // Find stable tag. const tags = await listRepositoryTags(signal); const stableTag = tags.find((tag) => tag.name === "stable"); if (!stableTag) { throw new ApiError("No stable tag found", 404); } // Now find all tags with the same image. const up = tags // Filter by digest. .filter((tag) => tag.digest === stableTag.digest) // Name should be a version number in a correct format (should start with a number or v and number). .filter(tag => /^v?\d+/.test(tag.name)) // And sort it in natural order (so that 1.1.10 is after 1.1.9). .sort(compare("name")); const last = up[up.length - 1]; // Panic if we don't have any tags that looks like version numbers. if (!last) { throw new ApiError("No stable image found", 404); } return { latestVersion: last.name, updatedAt: last.tag_last_pushed, isCritical: false, updateURL: Deps.DOCKER_IMAGE_SITE }; } catch (err) { // Make sure to throw only ApiErrors (cache depends on that). if (err instanceof ApiError) { throw err; } throw new ApiError(err.message, 500); } } // Shape of the data from the Docker Hub API. interface DockerTag { name: string; digest: string; tag_last_pushed: string; } interface DockerResponse { results: DockerTag[]; next: string|null; } // https://docs.docker.com/docker-hub/api/latest/#tag/repositories/ // paths/~1v2~1namespaces~1%7Bnamespace%7D~1repositories~1%7Brepository%7D~1tags/get async function listRepositoryTags(signal: AbortSignal): Promise{ const tags: DockerTag[] = []; // In case of rate limiting, we will retry the request 20 times. // This is for all pages, so we might hit the limit multiple times. let MAX_RETRIES = 20; const url = new URL(Deps.DOCKER_ENDPOINT); url.searchParams.set("page_size", "100"); let next: string|null = url.toString(); // We assume have a maximum of 100 000 tags, if that is not enough, we will have to change this. let MAX_LOOPS = 1000; while (next && MAX_LOOPS-- > 0) { const response = await fetch(next, {signal, timeout: Deps.REQUEST_TIMEOUT}); if (response.status === 429) { // We hit the rate limit, let's wait a bit and try again. await new Promise((resolve) => setTimeout(resolve, Deps.RETRY_TIMEOUT)); if (signal.aborted) { throw new Error("Aborted"); } if (MAX_RETRIES-- <= 0) { throw new Error("Too many retries"); } continue; } if (response.status !== 200) { throw new ApiError(await response.text(), response.status); } const json: DockerResponse = await response.json(); tags.push(...json.results); next = json.next; } if (MAX_LOOPS <= 0) { throw new Error("Too many tags found"); } return tags; } /** * Helper for sorting in natural order (1.1.10 is after 1.1.9). */ function compare(prop: keyof T) { return (a: T, b: T) => { return naturalCompare(a[prop], b[prop]); }; }