(core) Endpoint to report on the latest version of stable grist-core image

Summary:
New endpoint `/api/version` that returns latest version of stable docker image in format:
```
{"latestVersion":"1.1.12","
  updatedAt":"2024-03-06T06:28:25.752337Z","
  isCritical":false,
 "updateURL":"https://hub.docker.com/r/gristlabs/grist"
}
```

It connects to docker hub API and reads the version from the tag lists endpoint.
Stores telemetry passed from the client such us: current version, deployment type, installationId and others.

Test Plan: Added new test

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D4220
pull/936/head
Jarosław Sadziński 1 month ago
parent ddc28e327b
commit bddbcddbef

@ -28,7 +28,7 @@ type Comparator = (val1: any, val2: any) => number;
* https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/localeCompare
*/
const collator = new Intl.Collator(undefined, {numeric: true});
function naturalCompare(val1: any, val2: any) {
export function naturalCompare(val1: any, val2: any) {
if (typeof val1 === 'string' && typeof val2 === 'string') {
return collator.compare(val1, val2);
}

@ -1740,6 +1740,22 @@ export const TelemetryContracts: TelemetryContracts = {
},
},
},
checkedUpdateAPI: {
category: "SelfHosted",
description: 'Triggered when the app checks for updates.',
minimumTelemetryLevel: Level.limited,
retentionPeriod: 'indefinitely',
metadataContracts: {
installationId: {
description: 'The installation id of the client.',
dataType: 'string',
},
deploymentType: {
description: 'The deployment type of the client.',
dataType: 'string',
},
},
}
};
type TelemetryContracts = Record<TelemetryEvent, TelemetryEventContract>;
@ -1810,6 +1826,7 @@ export const TelemetryEvents = StringUnion(
'visitedForm',
'submittedForm',
'changedAccessRules',
'checkedUpdateAPI'
);
export type TelemetryEvent = typeof TelemetryEvents.type;
@ -1824,7 +1841,8 @@ type TelemetryEventCategory =
| 'TeamSite'
| 'ProductVisits'
| 'AccessRules'
| 'WidgetUsage';
| 'WidgetUsage'
| 'SelfHosted';
interface TelemetryEventContract {
description: string;

@ -67,6 +67,7 @@ import {TagChecker} from 'app/server/lib/TagChecker';
import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
import {startTestingHooks} from 'app/server/lib/TestingHooks';
import {getTestLoginSystem} from 'app/server/lib/TestLogin';
import {UpdateManager} from 'app/server/lib/UpdateManager';
import {addUploadRoute} from 'app/server/lib/uploads';
import {buildWidgetRepository, getWidgetsInPlugins, IWidgetRepository} from 'app/server/lib/WidgetRepository';
import {setupLocale} from 'app/server/localization';
@ -179,6 +180,7 @@ export class FlexServer implements GristServer {
// Set once ready() is called
private _isReady: boolean = false;
private _probes: BootProbes;
private _updateManager: UpdateManager;
constructor(public port: number, public name: string = 'flexServer',
public readonly options: FlexServerOptions = {}) {
@ -405,6 +407,11 @@ export class FlexServer implements GristServer {
return this._accessTokens;
}
public getUpdateManager() {
if (!this._updateManager) { throw new Error('no UpdateManager available'); }
return this._updateManager;
}
public sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise<void> {
if (!this._sendAppPage) { throw new Error('no _sendAppPage method available'); }
return this._sendAppPage(req, resp, options);
@ -880,6 +887,7 @@ export class FlexServer implements GristServer {
public async close() {
this._processMonitorStop?.();
await this._updateManager?.clear();
if (this.usage) { await this.usage.close(); }
if (this._hosts) { this._hosts.close(); }
if (this._dbManager) {
@ -1863,6 +1871,16 @@ export class FlexServer implements GristServer {
return process.env.GRIST_TEST_LOGIN ? getTestLoginSystem() : (this._getLoginSystem?.() || getLoginSystem());
}
public addUpdatesCheck() {
if (this._check('update')) { return; }
// For now we only are active for sass deployments.
if (this._deploymentType !== 'saas') { return; }
this._updateManager = new UpdateManager(this.app, this);
this._updateManager.addEndpoints();
}
// Adds endpoints that support imports and exports.
private _addSupportPaths(docAccessMiddleware: express.RequestHandler[]) {
if (!this._docWorker) { throw new Error("need DocWorker"); }

@ -0,0 +1,263 @@
import { ApiError } from "app/common/ApiError";
import { MapWithTTL } from "app/common/AsyncCreate";
import { GristDeploymentType } from "app/common/gristUrls";
import { naturalCompare } from "app/common/SortFunc";
import { RequestWithLogin } from "app/server/lib/Authorizer";
import { GristServer } from "app/server/lib/GristServer";
import { optIntegerParam, optStringParam } from "app/server/lib/requestUtils";
import { AbortController, AbortSignal } from 'node-abort-controller';
import type * as express from "express";
import fetch from "node-fetch";
import {expressWrap} from 'app/server/lib/expressWrap';
// URL to show to the client where the new version for docker based deployments can be found.
const DOCKER_IMAGE_SITE = "https://hub.docker.com/r/gristlabs/grist";
// URL to show to the client where the new version for docker based deployments can be found.
const DOCKER_ENDPOINT = process.env.GRIST_TEST_UPDATE_DOCKER_HUB_URL ||
"https://hub.docker.com/v2/namespaces/gristlabs/repositories/grist/tags";
// Timeout for the request to the external resource.
const REQUEST_TIMEOUT = optIntegerParam(process.env.GRIST_TEST_UPDATE_REQUEST_TIMEOUT, '') ?? 10000; // 10s
// Delay between retries in case of rate limiting.
const RETRY_TIMEOUT = optIntegerParam(process.env.GRIST_TEST_UPDATE_RETRY_TIMEOUT, '') ?? 4000; // 4s
// We cache the good result for an hour.
const GOOD_RESULT_TTL = optIntegerParam(process.env.GRIST_TEST_UPDATE_CHECK_TTL, '') ?? 60 * 60 * 1000; // 1h
// We cache the bad result errors from external resources for a minute.
const BAD_RESULT_TTL = optIntegerParam(process.env.GRIST_TEST_UPDATE_ERROR_TTL, '') ?? 60 * 1000; // 1m
// A hook for tests to override the default values.
export const Deps = {
DOCKER_IMAGE_SITE,
DOCKER_ENDPOINT,
REQUEST_TIMEOUT,
RETRY_TIMEOUT,
GOOD_RESULT_TTL,
BAD_RESULT_TTL,
};
export class UpdateManager {
// Cache for the latest version of the client.
private _latestVersion: MapWithTTL<
GristDeploymentType,
// We cache the promise, so that we can wait for the first request.
// This promise will always resolves, but can be resolved with an error.
Promise<ApiError|LatestVersion>
>;
private _abortController = new AbortController();
public constructor(
private _app: express.Application,
private _server: GristServer
) {
this._latestVersion = new MapWithTTL<GristDeploymentType, Promise<ApiError|LatestVersion>>(Deps.GOOD_RESULT_TTL);
}
public addEndpoints() {
// Make sure that config is ok, so that we are not surprised when client asks as about that.
if (Deps.DOCKER_ENDPOINT) {
try {
new URL(Deps.DOCKER_ENDPOINT);
} catch (err) {
throw new Error(
`Invalid value for GRIST_UPDATE_DOCKER_URL, expected URL: ${Deps.DOCKER_ENDPOINT}`
);
}
}
// Support both POST and GET requests.
this._app.use("/api/version", expressWrap(async (req, res) => {
// Get some telemetry from the body request.
const payload = (name: string) => req.body?.[name] ?? req.query[name];
// This is the most interesting part for us, to track installation ids and match them
// with the version of the client. Won't be send without telemetry opt in.
const installationId = optStringParam(
payload("installationId"),
"installationId"
);
// Current version of grist-core part of the client. Currently not used and not
// passed from the client.
// Deployment type of the client (we expect this to be 'core' for most of the cases).
const deploymentType = optStringParam(
payload("deploymentType"),
"deploymentType"
) as GristDeploymentType|undefined;
this._server
.getTelemetry()
.logEvent(req as RequestWithLogin, "checkedUpdateAPI", {
full: {
installationId,
deploymentType,
},
});
// For now we will just check the latest tag of docker stable image, assuming
// that this is what the client wants. In the future we might have different
// implementation based on the client deployment type.
const deploymentToCheck = 'core';
const versionChecker: VersionChecker = getLatestStableDockerVersion;
// To not spam the docker hub with requests, we will cache the good result for an hour.
// We are actually caching the promise, so subsequent requests will wait for the first one.
if (!this._latestVersion.has(deploymentToCheck)) {
const task = versionChecker(this._abortController.signal).catch(err => err);
this._latestVersion.set(deploymentToCheck, task);
}
const resData = await this._latestVersion.get(deploymentToCheck)!;
if (resData instanceof ApiError) {
// If the request has failed for any reason, we will throw the error to the client,
// but shorten the TTL to 1 minute, so that the next client will try after that time.
this._latestVersion.setWithCustomTTL(deploymentToCheck, Promise.resolve(resData), Deps.BAD_RESULT_TTL);
throw resData;
}
res.json(resData);
}));
}
public async clear() {
this._abortController.abort();
for (const task of this._latestVersion.values()) {
await task.catch(() => {});
}
this._latestVersion.clear();
// This function just clears cache and state, we should end with a fine state.
this._abortController = new AbortController();
}
}
/**
* JSON returned to the client (exported for tests).
*/
export interface LatestVersion {
/**
* Latest version of core component of the client.
*/
latestVersion: string;
/**
* If there were any critical updates after client's version. Undefined if
* we don't know client version or couldn't figure this out for some other reason.
*/
isCritical?: boolean;
/**
* Url where the client can download the latest version (if applicable)
*/
updateURL?: string;
/**
* When the latest version was updated (in ISO format).
*/
updatedAt?: string;
}
type VersionChecker = (signal: AbortSignal) => Promise<LatestVersion>;
/**
* Get the latest stable version of docker image from the hub.
*/
export async function getLatestStableDockerVersion(signal: AbortSignal): Promise<LatestVersion> {
try {
// Find stable tag.
const tags = await listRepositoryTags(signal);
const stableTag = tags.find((tag) => tag.name === "stable");
if (!stableTag) {
throw new ApiError("No stable tag found", 404);
}
// Now find all tags with the same image.
const up = tags
// Filter by digest.
.filter((tag) => tag.digest === stableTag.digest)
// Name should be a version number in a correct format (should start with a number or v and number).
.filter(tag => /^v?\d+/.test(tag.name))
// And sort it in natural order (so that 1.1.10 is after 1.1.9).
.sort(compare("name"));
const last = up[up.length - 1];
// Panic if we don't have any tags that looks like version numbers.
if (!last) {
throw new ApiError("No stable image found", 404);
}
return {
latestVersion: last.name,
updatedAt: last.tag_last_pushed,
isCritical: false,
updateURL: Deps.DOCKER_IMAGE_SITE
};
} catch (err) {
// Make sure to throw only ApiErrors (cache depends on that).
if (err instanceof ApiError) {
throw err;
}
throw new ApiError(err.message, 500);
}
}
// Shape of the data from the Docker Hub API.
interface DockerTag {
name: string;
digest: string;
tag_last_pushed: string;
}
interface DockerResponse {
results: DockerTag[];
next: string|null;
}
// https://docs.docker.com/docker-hub/api/latest/#tag/repositories/
// paths/~1v2~1namespaces~1%7Bnamespace%7D~1repositories~1%7Brepository%7D~1tags/get
async function listRepositoryTags(signal: AbortSignal): Promise<DockerTag[]>{
const tags: DockerTag[] = [];
// In case of rate limiting, we will retry the request 20 times.
// This is for all pages, so we might hit the limit multiple times.
let MAX_RETRIES = 20;
const url = new URL(Deps.DOCKER_ENDPOINT);
url.searchParams.set("page_size", "100");
let next: string|null = url.toString();
// We assume have a maximum of 100 000 tags, if that is not enough, we will have to change this.
let MAX_LOOPS = 1000;
while (next && MAX_LOOPS-- > 0) {
const response = await fetch(next, {signal, timeout: Deps.REQUEST_TIMEOUT});
if (response.status === 429) {
// We hit the rate limit, let's wait a bit and try again.
await new Promise((resolve) => setTimeout(resolve, Deps.RETRY_TIMEOUT));
if (signal.aborted) {
throw new Error("Aborted");
}
if (MAX_RETRIES-- <= 0) {
throw new Error("Too many retries");
}
continue;
}
if (response.status !== 200) {
throw new ApiError(await response.text(), response.status);
}
const json: DockerResponse = await response.json();
tags.push(...json.results);
next = json.next;
}
if (MAX_LOOPS <= 0) {
throw new Error("Too many tags found");
}
return tags;
}
/**
* Helper for sorting in natural order (1.1.10 is after 1.1.9).
*/
function compare<T>(prop: keyof T) {
return (a: T, b: T) => {
return naturalCompare(a[prop], b[prop]);
};
}

@ -106,6 +106,7 @@ export async function main(port: number, serverTypes: ServerType[],
server.addHealthCheck();
if (includeHome || includeApp) {
server.addBootPage();
server.addUpdatesCheck();
}
server.denyRequestsIfNotReady();

@ -0,0 +1,347 @@
import axios from "axios";
import * as chai from "chai";
import * as sinon from 'sinon';
import { configForUser } from "test/gen-server/testUtils";
import * as testUtils from "test/server/testUtils";
import { serveSomething, Serving } from "test/server/customUtil";
import { Deps, LatestVersion } from "app/server/lib/UpdateManager";
import { TestServer } from "test/gen-server/apiUtils";
import { delay } from "app/common/delay";
const assert = chai.assert;
let testServer: TestServer;
const stop = async () => {
await testServer?.stop();
testServer = null as any;
};
let homeUrl: string;
let dockerHub: Serving & { signal: () => Defer };
const chimpy = configForUser("Chimpy");
// Tests specific complex scenarios that may have previously resulted in wrong behavior.
describe("UpdateChecks", function () {
testUtils.setTmpLogLevel("error");
this.timeout("20s");
const sandbox = sinon.createSandbox();
before(async function () {
testUtils.EnvironmentSnapshot.push();
dockerHub = await dummyDockerHub();
assert.equal((await fetch(dockerHub.url + "/tags")).status, 200);
// Start the server with correct configuration.
Object.assign(process.env, {
GRIST_TEST_SERVER_DEPLOYMENT_TYPE: "saas",
});
sandbox.stub(Deps, "REQUEST_TIMEOUT").value(300);
sandbox.stub(Deps, "RETRY_TIMEOUT").value(400);
sandbox.stub(Deps, "GOOD_RESULT_TTL").value(500);
sandbox.stub(Deps, "BAD_RESULT_TTL").value(200);
sandbox.stub(Deps, "DOCKER_ENDPOINT").value(dockerHub.url + "/tags");
await startInProcess(this);
});
after(async function () {
sandbox.restore();
await dockerHub.shutdown();
await stop();
testUtils.EnvironmentSnapshot.pop();
});
afterEach(async function () {
await testServer.server.getUpdateManager().clear();
});
it("should read latest version as anonymous user in happy path", async function () {
setEndpoint(dockerHub.url + "/tags");
const resp = await axios.get(`${homeUrl}/api/version`);
assert.equal(resp.status, 200, `${homeUrl}/api/version`);
const result: LatestVersion = resp.data;
assert.equal(result.latestVersion, "10");
// Also works in post method.
const resp2 = await axios.post(`${homeUrl}/api/version`);
assert.equal(resp2.status, 200);
assert.deepEqual(resp2.data, result);
});
it("should read latest version as existing user", async function () {
setEndpoint(dockerHub.url + "/tags");
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(resp.status, 200);
const result: LatestVersion = resp.data;
assert.equal(result.latestVersion, "10");
});
it("passes errors to client", async function () {
setEndpoint(dockerHub.url + "/404");
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, { error: "Not Found" });
});
it("retries on 429", async function () {
setEndpoint(dockerHub.url + "/429");
// First make sure that mock works.
assert.equal((await fetch(dockerHub.url + "/429")).status, 200);
assert.equal((await fetch(dockerHub.url + "/429")).status, 429);
assert.equal((await fetch(dockerHub.url + "/429")).status, 200);
assert.equal((await fetch(dockerHub.url + "/429")).status, 429);
// Now make sure that 4 subsequent requests are successful.
const check = async () => {
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(resp.status, 200);
const result: LatestVersion = resp.data;
assert.equal(result.latestVersion, "10");
};
await check();
await check();
await check();
await check();
});
it("throws when receives html", async function () {
setEndpoint(dockerHub.url + "/html");
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(resp.status, 500);
});
it("caches data end errors", async function () {
setEndpoint(dockerHub.url + "/error");
const r1 = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(r1.status, 500);
assert.equal(r1.data.error, "1");
const r2 = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(r2.status, 500);
assert.equal(r2.data.error, "1"); // since errors are cached for 200ms.
await delay(300); // error is cached for 200ms
const r3 = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(r3.status, 500);
assert.equal(r3.data.error, "2"); // second error is different, but still cached for 200ms.
const r4 = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(r4.status, 500);
assert.equal(r4.data.error, "2");
await delay(300);
// Now we should get correct result, but it will be cached for 500ms.
const r5 = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(r5.status, 200);
assert.equal(r5.data.latestVersion, "3"); // first successful response is cached for 2 seconds.
const r6 = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(r6.status, 200);
assert.equal(r6.data.latestVersion, "3");
await delay(700);
const r7 = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(r7.status, 200);
assert.equal(r7.data.latestVersion, "4");
});
it("can stop server when hangs", async function () {
setEndpoint(dockerHub.url + "/hang");
const handCalled = dockerHub.signal();
const resp = axios
.get(`${homeUrl}/api/version`, chimpy)
.catch((err) => ({ status: 999, data: null }));
await handCalled;
await stop();
const result = await resp;
assert.equal(result.status, 500);
assert.match(result.data.error, /aborted/);
// Start server again, and make sure it works.
await startInProcess(this);
});
it("dosent starts for non saas deployment", async function () {
try {
testUtils.EnvironmentSnapshot.push();
Object.assign(process.env, {
GRIST_TEST_SERVER_DEPLOYMENT_TYPE: "core",
});
await stop();
await startInProcess(this);
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(resp.status, 404);
} finally {
testUtils.EnvironmentSnapshot.pop();
}
// Start normal one again.
await stop();
await startInProcess(this);
});
it("reports error when timeout happens", async function () {
setEndpoint(dockerHub.url + "/timeout");
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
assert.equal(resp.status, 500);
assert.match(resp.data.error, /timeout/);
});
});
async function dummyDockerHub() {
let odds = 0;
// We offer a way to signal when request is received.
// Test can add a dummy promise using signal() method, and it is resolved
// when any request is received.
const signals: Defer[] = [];
let errorCount = 0;
const tempServer = await serveSomething((app) => {
app.use((req, res, next) => {
signals.forEach((p) => p.resolve());
signals.length = 0;
next();
});
app.get("/404", (_, res) => res.status(404).send("Not Found").end());
app.get("/429", (_, res) => {
if (odds++ % 2) {
res.status(429).send("Too Many Requests");
} else {
res.json(SECOND_PAGE);
}
});
app.get("/timeout", (_, res) => {
setTimeout(() => res.status(200).json(SECOND_PAGE), 500);
});
app.get("/error", (_, res) => {
errorCount++;
// First 2 calls will return error, next will return numbers (3, 4, 5, 6, 7, 8, 9, 10)
if (errorCount <= 2) {
res.status(500).send(String(errorCount));
} else {
res.json(VERSION(errorCount));
}
});
app.get("/html", (_, res) => {
res.status(200).send("<html></html>");
});
app.get("/hang", () => {});
app.get("/tags", (_, res) => {
res.status(200).json(FIRST_PAGE(tempServer));
});
app.get("/next", (_, res) => {
res.status(200).json(SECOND_PAGE);
});
});
return Object.assign(tempServer, {
signal() {
const p = defer();
signals.push(p);
return p;
},
});
}
function setEndpoint(endpoint: string) {
sinon.stub(Deps, "DOCKER_ENDPOINT").value(endpoint);
}
async function startInProcess(context: Mocha.Context) {
testServer = new TestServer(context);
await testServer.start(["home"]);
homeUrl = testServer.serverUrl;
}
const VERSION = (i: number) => ({
results: [
{
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
name: "stable",
digest: "stable",
},
{
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
name: i.toString(),
digest: "stable",
},
],
count: 2,
next: null,
});
const SECOND_PAGE = {
results: [
{
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
name: "stable",
digest: "stable",
},
{
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
name: "latest",
digest: "latest",
},
{
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
name: "1",
digest: "latest",
},
{
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
name: "1",
digest: "stable",
},
{
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
name: "9",
digest: "stable",
},
{
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
name: "10",
digest: "stable",
},
],
count: 6,
next: null,
};
const FIRST_PAGE = (tempServer: Serving) => ({
results: [],
count: 0,
next: tempServer.url + "/next",
});
interface Defer {
then: Promise<void>["then"];
resolve: () => void;
reject: () => void;
}
const defer = () => {
let resolve: () => void;
let reject: () => void;
const promise = new Promise<void>((res, rej) => {
resolve = res;
reject = rej;
}).catch(() => {});
return {
then: promise.then.bind(promise),
resolve: resolve!,
reject: reject!,
};
};

@ -302,11 +302,26 @@ export async function readFixtureDoc(docName: string) {
// a class to store a snapshot of environment variables, can be reverted to by
// calling .restore()
export class EnvironmentSnapshot {
public static push() {
this._stack.push(new EnvironmentSnapshot());
}
public static pop() {
const snapshot = this._stack.pop();
if (!snapshot) {
throw new Error("EnvironmentSnapshot stack is empty");
}
snapshot.restore();
}
private static _stack: EnvironmentSnapshot[] = [];
private _oldEnv: NodeJS.ProcessEnv;
public constructor() {
this._oldEnv = clone(process.env);
}
// Reset environment variables.
public restore() {
Object.assign(process.env, this._oldEnv);

Loading…
Cancel
Save