mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) Endpoint to report on the latest version of stable grist-core image
Summary: New endpoint `/api/version` that returns latest version of stable docker image in format: ``` {"latestVersion":"1.1.12"," updatedAt":"2024-03-06T06:28:25.752337Z"," isCritical":false, "updateURL":"https://hub.docker.com/r/gristlabs/grist" } ``` It connects to docker hub API and reads the version from the tag lists endpoint. Stores telemetry passed from the client such us: current version, deployment type, installationId and others. Test Plan: Added new test Reviewers: georgegevoian Reviewed By: georgegevoian Differential Revision: https://phab.getgrist.com/D4220
This commit is contained in:
parent
ddc28e327b
commit
bddbcddbef
@ -28,7 +28,7 @@ type Comparator = (val1: any, val2: any) => number;
|
|||||||
* https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/localeCompare
|
* https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/localeCompare
|
||||||
*/
|
*/
|
||||||
const collator = new Intl.Collator(undefined, {numeric: true});
|
const collator = new Intl.Collator(undefined, {numeric: true});
|
||||||
function naturalCompare(val1: any, val2: any) {
|
export function naturalCompare(val1: any, val2: any) {
|
||||||
if (typeof val1 === 'string' && typeof val2 === 'string') {
|
if (typeof val1 === 'string' && typeof val2 === 'string') {
|
||||||
return collator.compare(val1, val2);
|
return collator.compare(val1, val2);
|
||||||
}
|
}
|
||||||
|
@ -1740,6 +1740,22 @@ export const TelemetryContracts: TelemetryContracts = {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
checkedUpdateAPI: {
|
||||||
|
category: "SelfHosted",
|
||||||
|
description: 'Triggered when the app checks for updates.',
|
||||||
|
minimumTelemetryLevel: Level.limited,
|
||||||
|
retentionPeriod: 'indefinitely',
|
||||||
|
metadataContracts: {
|
||||||
|
installationId: {
|
||||||
|
description: 'The installation id of the client.',
|
||||||
|
dataType: 'string',
|
||||||
|
},
|
||||||
|
deploymentType: {
|
||||||
|
description: 'The deployment type of the client.',
|
||||||
|
dataType: 'string',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
type TelemetryContracts = Record<TelemetryEvent, TelemetryEventContract>;
|
type TelemetryContracts = Record<TelemetryEvent, TelemetryEventContract>;
|
||||||
@ -1810,6 +1826,7 @@ export const TelemetryEvents = StringUnion(
|
|||||||
'visitedForm',
|
'visitedForm',
|
||||||
'submittedForm',
|
'submittedForm',
|
||||||
'changedAccessRules',
|
'changedAccessRules',
|
||||||
|
'checkedUpdateAPI'
|
||||||
);
|
);
|
||||||
export type TelemetryEvent = typeof TelemetryEvents.type;
|
export type TelemetryEvent = typeof TelemetryEvents.type;
|
||||||
|
|
||||||
@ -1824,7 +1841,8 @@ type TelemetryEventCategory =
|
|||||||
| 'TeamSite'
|
| 'TeamSite'
|
||||||
| 'ProductVisits'
|
| 'ProductVisits'
|
||||||
| 'AccessRules'
|
| 'AccessRules'
|
||||||
| 'WidgetUsage';
|
| 'WidgetUsage'
|
||||||
|
| 'SelfHosted';
|
||||||
|
|
||||||
interface TelemetryEventContract {
|
interface TelemetryEventContract {
|
||||||
description: string;
|
description: string;
|
||||||
|
@ -67,6 +67,7 @@ import {TagChecker} from 'app/server/lib/TagChecker';
|
|||||||
import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
|
import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
|
||||||
import {startTestingHooks} from 'app/server/lib/TestingHooks';
|
import {startTestingHooks} from 'app/server/lib/TestingHooks';
|
||||||
import {getTestLoginSystem} from 'app/server/lib/TestLogin';
|
import {getTestLoginSystem} from 'app/server/lib/TestLogin';
|
||||||
|
import {UpdateManager} from 'app/server/lib/UpdateManager';
|
||||||
import {addUploadRoute} from 'app/server/lib/uploads';
|
import {addUploadRoute} from 'app/server/lib/uploads';
|
||||||
import {buildWidgetRepository, getWidgetsInPlugins, IWidgetRepository} from 'app/server/lib/WidgetRepository';
|
import {buildWidgetRepository, getWidgetsInPlugins, IWidgetRepository} from 'app/server/lib/WidgetRepository';
|
||||||
import {setupLocale} from 'app/server/localization';
|
import {setupLocale} from 'app/server/localization';
|
||||||
@ -179,6 +180,7 @@ export class FlexServer implements GristServer {
|
|||||||
// Set once ready() is called
|
// Set once ready() is called
|
||||||
private _isReady: boolean = false;
|
private _isReady: boolean = false;
|
||||||
private _probes: BootProbes;
|
private _probes: BootProbes;
|
||||||
|
private _updateManager: UpdateManager;
|
||||||
|
|
||||||
constructor(public port: number, public name: string = 'flexServer',
|
constructor(public port: number, public name: string = 'flexServer',
|
||||||
public readonly options: FlexServerOptions = {}) {
|
public readonly options: FlexServerOptions = {}) {
|
||||||
@ -405,6 +407,11 @@ export class FlexServer implements GristServer {
|
|||||||
return this._accessTokens;
|
return this._accessTokens;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public getUpdateManager() {
|
||||||
|
if (!this._updateManager) { throw new Error('no UpdateManager available'); }
|
||||||
|
return this._updateManager;
|
||||||
|
}
|
||||||
|
|
||||||
public sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise<void> {
|
public sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise<void> {
|
||||||
if (!this._sendAppPage) { throw new Error('no _sendAppPage method available'); }
|
if (!this._sendAppPage) { throw new Error('no _sendAppPage method available'); }
|
||||||
return this._sendAppPage(req, resp, options);
|
return this._sendAppPage(req, resp, options);
|
||||||
@ -880,6 +887,7 @@ export class FlexServer implements GristServer {
|
|||||||
|
|
||||||
public async close() {
|
public async close() {
|
||||||
this._processMonitorStop?.();
|
this._processMonitorStop?.();
|
||||||
|
await this._updateManager?.clear();
|
||||||
if (this.usage) { await this.usage.close(); }
|
if (this.usage) { await this.usage.close(); }
|
||||||
if (this._hosts) { this._hosts.close(); }
|
if (this._hosts) { this._hosts.close(); }
|
||||||
if (this._dbManager) {
|
if (this._dbManager) {
|
||||||
@ -1863,6 +1871,16 @@ export class FlexServer implements GristServer {
|
|||||||
return process.env.GRIST_TEST_LOGIN ? getTestLoginSystem() : (this._getLoginSystem?.() || getLoginSystem());
|
return process.env.GRIST_TEST_LOGIN ? getTestLoginSystem() : (this._getLoginSystem?.() || getLoginSystem());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public addUpdatesCheck() {
|
||||||
|
if (this._check('update')) { return; }
|
||||||
|
|
||||||
|
// For now we only are active for sass deployments.
|
||||||
|
if (this._deploymentType !== 'saas') { return; }
|
||||||
|
|
||||||
|
this._updateManager = new UpdateManager(this.app, this);
|
||||||
|
this._updateManager.addEndpoints();
|
||||||
|
}
|
||||||
|
|
||||||
// Adds endpoints that support imports and exports.
|
// Adds endpoints that support imports and exports.
|
||||||
private _addSupportPaths(docAccessMiddleware: express.RequestHandler[]) {
|
private _addSupportPaths(docAccessMiddleware: express.RequestHandler[]) {
|
||||||
if (!this._docWorker) { throw new Error("need DocWorker"); }
|
if (!this._docWorker) { throw new Error("need DocWorker"); }
|
||||||
|
263
app/server/lib/UpdateManager.ts
Normal file
263
app/server/lib/UpdateManager.ts
Normal file
@ -0,0 +1,263 @@
|
|||||||
|
import { ApiError } from "app/common/ApiError";
|
||||||
|
import { MapWithTTL } from "app/common/AsyncCreate";
|
||||||
|
import { GristDeploymentType } from "app/common/gristUrls";
|
||||||
|
import { naturalCompare } from "app/common/SortFunc";
|
||||||
|
import { RequestWithLogin } from "app/server/lib/Authorizer";
|
||||||
|
import { GristServer } from "app/server/lib/GristServer";
|
||||||
|
import { optIntegerParam, optStringParam } from "app/server/lib/requestUtils";
|
||||||
|
import { AbortController, AbortSignal } from 'node-abort-controller';
|
||||||
|
import type * as express from "express";
|
||||||
|
import fetch from "node-fetch";
|
||||||
|
import {expressWrap} from 'app/server/lib/expressWrap';
|
||||||
|
|
||||||
|
|
||||||
|
// URL to show to the client where the new version for docker based deployments can be found.
|
||||||
|
const DOCKER_IMAGE_SITE = "https://hub.docker.com/r/gristlabs/grist";
|
||||||
|
|
||||||
|
// URL to show to the client where the new version for docker based deployments can be found.
|
||||||
|
const DOCKER_ENDPOINT = process.env.GRIST_TEST_UPDATE_DOCKER_HUB_URL ||
|
||||||
|
"https://hub.docker.com/v2/namespaces/gristlabs/repositories/grist/tags";
|
||||||
|
// Timeout for the request to the external resource.
|
||||||
|
const REQUEST_TIMEOUT = optIntegerParam(process.env.GRIST_TEST_UPDATE_REQUEST_TIMEOUT, '') ?? 10000; // 10s
|
||||||
|
// Delay between retries in case of rate limiting.
|
||||||
|
const RETRY_TIMEOUT = optIntegerParam(process.env.GRIST_TEST_UPDATE_RETRY_TIMEOUT, '') ?? 4000; // 4s
|
||||||
|
// We cache the good result for an hour.
|
||||||
|
const GOOD_RESULT_TTL = optIntegerParam(process.env.GRIST_TEST_UPDATE_CHECK_TTL, '') ?? 60 * 60 * 1000; // 1h
|
||||||
|
// We cache the bad result errors from external resources for a minute.
|
||||||
|
const BAD_RESULT_TTL = optIntegerParam(process.env.GRIST_TEST_UPDATE_ERROR_TTL, '') ?? 60 * 1000; // 1m
|
||||||
|
|
||||||
|
// A hook for tests to override the default values.
|
||||||
|
export const Deps = {
|
||||||
|
DOCKER_IMAGE_SITE,
|
||||||
|
DOCKER_ENDPOINT,
|
||||||
|
REQUEST_TIMEOUT,
|
||||||
|
RETRY_TIMEOUT,
|
||||||
|
GOOD_RESULT_TTL,
|
||||||
|
BAD_RESULT_TTL,
|
||||||
|
};
|
||||||
|
|
||||||
|
export class UpdateManager {
|
||||||
|
|
||||||
|
// Cache for the latest version of the client.
|
||||||
|
private _latestVersion: MapWithTTL<
|
||||||
|
GristDeploymentType,
|
||||||
|
// We cache the promise, so that we can wait for the first request.
|
||||||
|
// This promise will always resolves, but can be resolved with an error.
|
||||||
|
Promise<ApiError|LatestVersion>
|
||||||
|
>;
|
||||||
|
|
||||||
|
private _abortController = new AbortController();
|
||||||
|
|
||||||
|
public constructor(
|
||||||
|
private _app: express.Application,
|
||||||
|
private _server: GristServer
|
||||||
|
) {
|
||||||
|
this._latestVersion = new MapWithTTL<GristDeploymentType, Promise<ApiError|LatestVersion>>(Deps.GOOD_RESULT_TTL);
|
||||||
|
}
|
||||||
|
|
||||||
|
public addEndpoints() {
|
||||||
|
// Make sure that config is ok, so that we are not surprised when client asks as about that.
|
||||||
|
if (Deps.DOCKER_ENDPOINT) {
|
||||||
|
try {
|
||||||
|
new URL(Deps.DOCKER_ENDPOINT);
|
||||||
|
} catch (err) {
|
||||||
|
throw new Error(
|
||||||
|
`Invalid value for GRIST_UPDATE_DOCKER_URL, expected URL: ${Deps.DOCKER_ENDPOINT}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Support both POST and GET requests.
|
||||||
|
this._app.use("/api/version", expressWrap(async (req, res) => {
|
||||||
|
// Get some telemetry from the body request.
|
||||||
|
const payload = (name: string) => req.body?.[name] ?? req.query[name];
|
||||||
|
|
||||||
|
// This is the most interesting part for us, to track installation ids and match them
|
||||||
|
// with the version of the client. Won't be send without telemetry opt in.
|
||||||
|
const installationId = optStringParam(
|
||||||
|
payload("installationId"),
|
||||||
|
"installationId"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Current version of grist-core part of the client. Currently not used and not
|
||||||
|
// passed from the client.
|
||||||
|
|
||||||
|
// Deployment type of the client (we expect this to be 'core' for most of the cases).
|
||||||
|
const deploymentType = optStringParam(
|
||||||
|
payload("deploymentType"),
|
||||||
|
"deploymentType"
|
||||||
|
) as GristDeploymentType|undefined;
|
||||||
|
|
||||||
|
this._server
|
||||||
|
.getTelemetry()
|
||||||
|
.logEvent(req as RequestWithLogin, "checkedUpdateAPI", {
|
||||||
|
full: {
|
||||||
|
installationId,
|
||||||
|
deploymentType,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// For now we will just check the latest tag of docker stable image, assuming
|
||||||
|
// that this is what the client wants. In the future we might have different
|
||||||
|
// implementation based on the client deployment type.
|
||||||
|
const deploymentToCheck = 'core';
|
||||||
|
const versionChecker: VersionChecker = getLatestStableDockerVersion;
|
||||||
|
|
||||||
|
// To not spam the docker hub with requests, we will cache the good result for an hour.
|
||||||
|
// We are actually caching the promise, so subsequent requests will wait for the first one.
|
||||||
|
if (!this._latestVersion.has(deploymentToCheck)) {
|
||||||
|
const task = versionChecker(this._abortController.signal).catch(err => err);
|
||||||
|
this._latestVersion.set(deploymentToCheck, task);
|
||||||
|
}
|
||||||
|
const resData = await this._latestVersion.get(deploymentToCheck)!;
|
||||||
|
if (resData instanceof ApiError) {
|
||||||
|
// If the request has failed for any reason, we will throw the error to the client,
|
||||||
|
// but shorten the TTL to 1 minute, so that the next client will try after that time.
|
||||||
|
this._latestVersion.setWithCustomTTL(deploymentToCheck, Promise.resolve(resData), Deps.BAD_RESULT_TTL);
|
||||||
|
throw resData;
|
||||||
|
}
|
||||||
|
res.json(resData);
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
public async clear() {
|
||||||
|
this._abortController.abort();
|
||||||
|
for (const task of this._latestVersion.values()) {
|
||||||
|
await task.catch(() => {});
|
||||||
|
}
|
||||||
|
this._latestVersion.clear();
|
||||||
|
|
||||||
|
// This function just clears cache and state, we should end with a fine state.
|
||||||
|
this._abortController = new AbortController();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JSON returned to the client (exported for tests).
|
||||||
|
*/
|
||||||
|
export interface LatestVersion {
|
||||||
|
/**
|
||||||
|
* Latest version of core component of the client.
|
||||||
|
*/
|
||||||
|
latestVersion: string;
|
||||||
|
/**
|
||||||
|
* If there were any critical updates after client's version. Undefined if
|
||||||
|
* we don't know client version or couldn't figure this out for some other reason.
|
||||||
|
*/
|
||||||
|
isCritical?: boolean;
|
||||||
|
/**
|
||||||
|
* Url where the client can download the latest version (if applicable)
|
||||||
|
*/
|
||||||
|
updateURL?: string;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* When the latest version was updated (in ISO format).
|
||||||
|
*/
|
||||||
|
updatedAt?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
type VersionChecker = (signal: AbortSignal) => Promise<LatestVersion>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the latest stable version of docker image from the hub.
|
||||||
|
*/
|
||||||
|
export async function getLatestStableDockerVersion(signal: AbortSignal): Promise<LatestVersion> {
|
||||||
|
try {
|
||||||
|
// Find stable tag.
|
||||||
|
const tags = await listRepositoryTags(signal);
|
||||||
|
const stableTag = tags.find((tag) => tag.name === "stable");
|
||||||
|
if (!stableTag) {
|
||||||
|
throw new ApiError("No stable tag found", 404);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now find all tags with the same image.
|
||||||
|
const up = tags
|
||||||
|
// Filter by digest.
|
||||||
|
.filter((tag) => tag.digest === stableTag.digest)
|
||||||
|
// Name should be a version number in a correct format (should start with a number or v and number).
|
||||||
|
.filter(tag => /^v?\d+/.test(tag.name))
|
||||||
|
// And sort it in natural order (so that 1.1.10 is after 1.1.9).
|
||||||
|
.sort(compare("name"));
|
||||||
|
|
||||||
|
const last = up[up.length - 1];
|
||||||
|
// Panic if we don't have any tags that looks like version numbers.
|
||||||
|
if (!last) {
|
||||||
|
throw new ApiError("No stable image found", 404);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
latestVersion: last.name,
|
||||||
|
updatedAt: last.tag_last_pushed,
|
||||||
|
isCritical: false,
|
||||||
|
updateURL: Deps.DOCKER_IMAGE_SITE
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
// Make sure to throw only ApiErrors (cache depends on that).
|
||||||
|
if (err instanceof ApiError) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
throw new ApiError(err.message, 500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shape of the data from the Docker Hub API.
|
||||||
|
interface DockerTag {
|
||||||
|
name: string;
|
||||||
|
digest: string;
|
||||||
|
tag_last_pushed: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DockerResponse {
|
||||||
|
results: DockerTag[];
|
||||||
|
next: string|null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://docs.docker.com/docker-hub/api/latest/#tag/repositories/
|
||||||
|
// paths/~1v2~1namespaces~1%7Bnamespace%7D~1repositories~1%7Brepository%7D~1tags/get
|
||||||
|
async function listRepositoryTags(signal: AbortSignal): Promise<DockerTag[]>{
|
||||||
|
const tags: DockerTag[] = [];
|
||||||
|
|
||||||
|
// In case of rate limiting, we will retry the request 20 times.
|
||||||
|
// This is for all pages, so we might hit the limit multiple times.
|
||||||
|
let MAX_RETRIES = 20;
|
||||||
|
|
||||||
|
const url = new URL(Deps.DOCKER_ENDPOINT);
|
||||||
|
url.searchParams.set("page_size", "100");
|
||||||
|
let next: string|null = url.toString();
|
||||||
|
|
||||||
|
// We assume have a maximum of 100 000 tags, if that is not enough, we will have to change this.
|
||||||
|
let MAX_LOOPS = 1000;
|
||||||
|
|
||||||
|
while (next && MAX_LOOPS-- > 0) {
|
||||||
|
const response = await fetch(next, {signal, timeout: Deps.REQUEST_TIMEOUT});
|
||||||
|
if (response.status === 429) {
|
||||||
|
// We hit the rate limit, let's wait a bit and try again.
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, Deps.RETRY_TIMEOUT));
|
||||||
|
if (signal.aborted) {
|
||||||
|
throw new Error("Aborted");
|
||||||
|
}
|
||||||
|
if (MAX_RETRIES-- <= 0) {
|
||||||
|
throw new Error("Too many retries");
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (response.status !== 200) {
|
||||||
|
throw new ApiError(await response.text(), response.status);
|
||||||
|
}
|
||||||
|
const json: DockerResponse = await response.json();
|
||||||
|
tags.push(...json.results);
|
||||||
|
next = json.next;
|
||||||
|
}
|
||||||
|
if (MAX_LOOPS <= 0) {
|
||||||
|
throw new Error("Too many tags found");
|
||||||
|
}
|
||||||
|
return tags;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper for sorting in natural order (1.1.10 is after 1.1.9).
|
||||||
|
*/
|
||||||
|
function compare<T>(prop: keyof T) {
|
||||||
|
return (a: T, b: T) => {
|
||||||
|
return naturalCompare(a[prop], b[prop]);
|
||||||
|
};
|
||||||
|
}
|
@ -106,6 +106,7 @@ export async function main(port: number, serverTypes: ServerType[],
|
|||||||
server.addHealthCheck();
|
server.addHealthCheck();
|
||||||
if (includeHome || includeApp) {
|
if (includeHome || includeApp) {
|
||||||
server.addBootPage();
|
server.addBootPage();
|
||||||
|
server.addUpdatesCheck();
|
||||||
}
|
}
|
||||||
server.denyRequestsIfNotReady();
|
server.denyRequestsIfNotReady();
|
||||||
|
|
||||||
|
347
test/gen-server/UpdateChecks.ts
Normal file
347
test/gen-server/UpdateChecks.ts
Normal file
@ -0,0 +1,347 @@
|
|||||||
|
import axios from "axios";
|
||||||
|
import * as chai from "chai";
|
||||||
|
import * as sinon from 'sinon';
|
||||||
|
|
||||||
|
import { configForUser } from "test/gen-server/testUtils";
|
||||||
|
import * as testUtils from "test/server/testUtils";
|
||||||
|
import { serveSomething, Serving } from "test/server/customUtil";
|
||||||
|
import { Deps, LatestVersion } from "app/server/lib/UpdateManager";
|
||||||
|
import { TestServer } from "test/gen-server/apiUtils";
|
||||||
|
import { delay } from "app/common/delay";
|
||||||
|
|
||||||
|
const assert = chai.assert;
|
||||||
|
|
||||||
|
let testServer: TestServer;
|
||||||
|
|
||||||
|
const stop = async () => {
|
||||||
|
await testServer?.stop();
|
||||||
|
testServer = null as any;
|
||||||
|
};
|
||||||
|
|
||||||
|
let homeUrl: string;
|
||||||
|
let dockerHub: Serving & { signal: () => Defer };
|
||||||
|
|
||||||
|
const chimpy = configForUser("Chimpy");
|
||||||
|
|
||||||
|
// Tests specific complex scenarios that may have previously resulted in wrong behavior.
|
||||||
|
describe("UpdateChecks", function () {
|
||||||
|
testUtils.setTmpLogLevel("error");
|
||||||
|
|
||||||
|
this.timeout("20s");
|
||||||
|
|
||||||
|
const sandbox = sinon.createSandbox();
|
||||||
|
|
||||||
|
before(async function () {
|
||||||
|
testUtils.EnvironmentSnapshot.push();
|
||||||
|
dockerHub = await dummyDockerHub();
|
||||||
|
assert.equal((await fetch(dockerHub.url + "/tags")).status, 200);
|
||||||
|
|
||||||
|
// Start the server with correct configuration.
|
||||||
|
Object.assign(process.env, {
|
||||||
|
GRIST_TEST_SERVER_DEPLOYMENT_TYPE: "saas",
|
||||||
|
});
|
||||||
|
sandbox.stub(Deps, "REQUEST_TIMEOUT").value(300);
|
||||||
|
sandbox.stub(Deps, "RETRY_TIMEOUT").value(400);
|
||||||
|
sandbox.stub(Deps, "GOOD_RESULT_TTL").value(500);
|
||||||
|
sandbox.stub(Deps, "BAD_RESULT_TTL").value(200);
|
||||||
|
sandbox.stub(Deps, "DOCKER_ENDPOINT").value(dockerHub.url + "/tags");
|
||||||
|
|
||||||
|
await startInProcess(this);
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async function () {
|
||||||
|
sandbox.restore();
|
||||||
|
await dockerHub.shutdown();
|
||||||
|
await stop();
|
||||||
|
testUtils.EnvironmentSnapshot.pop();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async function () {
|
||||||
|
await testServer.server.getUpdateManager().clear();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should read latest version as anonymous user in happy path", async function () {
|
||||||
|
setEndpoint(dockerHub.url + "/tags");
|
||||||
|
const resp = await axios.get(`${homeUrl}/api/version`);
|
||||||
|
assert.equal(resp.status, 200, `${homeUrl}/api/version`);
|
||||||
|
const result: LatestVersion = resp.data;
|
||||||
|
assert.equal(result.latestVersion, "10");
|
||||||
|
|
||||||
|
// Also works in post method.
|
||||||
|
const resp2 = await axios.post(`${homeUrl}/api/version`);
|
||||||
|
assert.equal(resp2.status, 200);
|
||||||
|
assert.deepEqual(resp2.data, result);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should read latest version as existing user", async function () {
|
||||||
|
setEndpoint(dockerHub.url + "/tags");
|
||||||
|
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(resp.status, 200);
|
||||||
|
const result: LatestVersion = resp.data;
|
||||||
|
assert.equal(result.latestVersion, "10");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes errors to client", async function () {
|
||||||
|
setEndpoint(dockerHub.url + "/404");
|
||||||
|
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(resp.status, 404);
|
||||||
|
assert.deepEqual(resp.data, { error: "Not Found" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries on 429", async function () {
|
||||||
|
setEndpoint(dockerHub.url + "/429");
|
||||||
|
|
||||||
|
// First make sure that mock works.
|
||||||
|
assert.equal((await fetch(dockerHub.url + "/429")).status, 200);
|
||||||
|
assert.equal((await fetch(dockerHub.url + "/429")).status, 429);
|
||||||
|
assert.equal((await fetch(dockerHub.url + "/429")).status, 200);
|
||||||
|
assert.equal((await fetch(dockerHub.url + "/429")).status, 429);
|
||||||
|
|
||||||
|
// Now make sure that 4 subsequent requests are successful.
|
||||||
|
const check = async () => {
|
||||||
|
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(resp.status, 200);
|
||||||
|
const result: LatestVersion = resp.data;
|
||||||
|
assert.equal(result.latestVersion, "10");
|
||||||
|
};
|
||||||
|
|
||||||
|
await check();
|
||||||
|
await check();
|
||||||
|
await check();
|
||||||
|
await check();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws when receives html", async function () {
|
||||||
|
setEndpoint(dockerHub.url + "/html");
|
||||||
|
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(resp.status, 500);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("caches data end errors", async function () {
|
||||||
|
setEndpoint(dockerHub.url + "/error");
|
||||||
|
const r1 = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(r1.status, 500);
|
||||||
|
assert.equal(r1.data.error, "1");
|
||||||
|
|
||||||
|
const r2 = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(r2.status, 500);
|
||||||
|
assert.equal(r2.data.error, "1"); // since errors are cached for 200ms.
|
||||||
|
|
||||||
|
await delay(300); // error is cached for 200ms
|
||||||
|
|
||||||
|
const r3 = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(r3.status, 500);
|
||||||
|
assert.equal(r3.data.error, "2"); // second error is different, but still cached for 200ms.
|
||||||
|
|
||||||
|
const r4 = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(r4.status, 500);
|
||||||
|
assert.equal(r4.data.error, "2");
|
||||||
|
|
||||||
|
await delay(300);
|
||||||
|
|
||||||
|
// Now we should get correct result, but it will be cached for 500ms.
|
||||||
|
|
||||||
|
const r5 = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(r5.status, 200);
|
||||||
|
assert.equal(r5.data.latestVersion, "3"); // first successful response is cached for 2 seconds.
|
||||||
|
|
||||||
|
const r6 = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(r6.status, 200);
|
||||||
|
assert.equal(r6.data.latestVersion, "3");
|
||||||
|
|
||||||
|
await delay(700);
|
||||||
|
|
||||||
|
const r7 = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(r7.status, 200);
|
||||||
|
assert.equal(r7.data.latestVersion, "4");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("can stop server when hangs", async function () {
|
||||||
|
setEndpoint(dockerHub.url + "/hang");
|
||||||
|
const handCalled = dockerHub.signal();
|
||||||
|
const resp = axios
|
||||||
|
.get(`${homeUrl}/api/version`, chimpy)
|
||||||
|
.catch((err) => ({ status: 999, data: null }));
|
||||||
|
await handCalled;
|
||||||
|
await stop();
|
||||||
|
const result = await resp;
|
||||||
|
assert.equal(result.status, 500);
|
||||||
|
assert.match(result.data.error, /aborted/);
|
||||||
|
// Start server again, and make sure it works.
|
||||||
|
await startInProcess(this);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("dosent starts for non saas deployment", async function () {
|
||||||
|
try {
|
||||||
|
testUtils.EnvironmentSnapshot.push();
|
||||||
|
Object.assign(process.env, {
|
||||||
|
GRIST_TEST_SERVER_DEPLOYMENT_TYPE: "core",
|
||||||
|
});
|
||||||
|
await stop();
|
||||||
|
await startInProcess(this);
|
||||||
|
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(resp.status, 404);
|
||||||
|
} finally {
|
||||||
|
testUtils.EnvironmentSnapshot.pop();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start normal one again.
|
||||||
|
await stop();
|
||||||
|
await startInProcess(this);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports error when timeout happens", async function () {
|
||||||
|
setEndpoint(dockerHub.url + "/timeout");
|
||||||
|
const resp = await axios.get(`${homeUrl}/api/version`, chimpy);
|
||||||
|
assert.equal(resp.status, 500);
|
||||||
|
assert.match(resp.data.error, /timeout/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
async function dummyDockerHub() {
|
||||||
|
let odds = 0;
|
||||||
|
|
||||||
|
// We offer a way to signal when request is received.
|
||||||
|
// Test can add a dummy promise using signal() method, and it is resolved
|
||||||
|
// when any request is received.
|
||||||
|
const signals: Defer[] = [];
|
||||||
|
let errorCount = 0;
|
||||||
|
|
||||||
|
const tempServer = await serveSomething((app) => {
|
||||||
|
app.use((req, res, next) => {
|
||||||
|
signals.forEach((p) => p.resolve());
|
||||||
|
signals.length = 0;
|
||||||
|
next();
|
||||||
|
});
|
||||||
|
app.get("/404", (_, res) => res.status(404).send("Not Found").end());
|
||||||
|
app.get("/429", (_, res) => {
|
||||||
|
if (odds++ % 2) {
|
||||||
|
res.status(429).send("Too Many Requests");
|
||||||
|
} else {
|
||||||
|
res.json(SECOND_PAGE);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
app.get("/timeout", (_, res) => {
|
||||||
|
setTimeout(() => res.status(200).json(SECOND_PAGE), 500);
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get("/error", (_, res) => {
|
||||||
|
errorCount++;
|
||||||
|
// First 2 calls will return error, next will return numbers (3, 4, 5, 6, 7, 8, 9, 10)
|
||||||
|
if (errorCount <= 2) {
|
||||||
|
res.status(500).send(String(errorCount));
|
||||||
|
} else {
|
||||||
|
res.json(VERSION(errorCount));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get("/html", (_, res) => {
|
||||||
|
res.status(200).send("<html></html>");
|
||||||
|
});
|
||||||
|
app.get("/hang", () => {});
|
||||||
|
app.get("/tags", (_, res) => {
|
||||||
|
res.status(200).json(FIRST_PAGE(tempServer));
|
||||||
|
});
|
||||||
|
app.get("/next", (_, res) => {
|
||||||
|
res.status(200).json(SECOND_PAGE);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return Object.assign(tempServer, {
|
||||||
|
signal() {
|
||||||
|
const p = defer();
|
||||||
|
signals.push(p);
|
||||||
|
return p;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function setEndpoint(endpoint: string) {
|
||||||
|
sinon.stub(Deps, "DOCKER_ENDPOINT").value(endpoint);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startInProcess(context: Mocha.Context) {
|
||||||
|
testServer = new TestServer(context);
|
||||||
|
await testServer.start(["home"]);
|
||||||
|
homeUrl = testServer.serverUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
const VERSION = (i: number) => ({
|
||||||
|
results: [
|
||||||
|
{
|
||||||
|
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
|
||||||
|
name: "stable",
|
||||||
|
digest: "stable",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
|
||||||
|
name: i.toString(),
|
||||||
|
digest: "stable",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
count: 2,
|
||||||
|
next: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const SECOND_PAGE = {
|
||||||
|
results: [
|
||||||
|
{
|
||||||
|
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
|
||||||
|
name: "stable",
|
||||||
|
digest: "stable",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
|
||||||
|
name: "latest",
|
||||||
|
digest: "latest",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
|
||||||
|
name: "1",
|
||||||
|
digest: "latest",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
|
||||||
|
name: "1",
|
||||||
|
digest: "stable",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
|
||||||
|
name: "9",
|
||||||
|
digest: "stable",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
tag_last_pushed: "2024-03-26T07:11:01.272113Z",
|
||||||
|
name: "10",
|
||||||
|
digest: "stable",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
count: 6,
|
||||||
|
next: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
const FIRST_PAGE = (tempServer: Serving) => ({
|
||||||
|
results: [],
|
||||||
|
count: 0,
|
||||||
|
next: tempServer.url + "/next",
|
||||||
|
});
|
||||||
|
|
||||||
|
interface Defer {
|
||||||
|
then: Promise<void>["then"];
|
||||||
|
resolve: () => void;
|
||||||
|
reject: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const defer = () => {
|
||||||
|
let resolve: () => void;
|
||||||
|
let reject: () => void;
|
||||||
|
const promise = new Promise<void>((res, rej) => {
|
||||||
|
resolve = res;
|
||||||
|
reject = rej;
|
||||||
|
}).catch(() => {});
|
||||||
|
return {
|
||||||
|
then: promise.then.bind(promise),
|
||||||
|
resolve: resolve!,
|
||||||
|
reject: reject!,
|
||||||
|
};
|
||||||
|
};
|
@ -302,11 +302,26 @@ export async function readFixtureDoc(docName: string) {
|
|||||||
// a class to store a snapshot of environment variables, can be reverted to by
|
// a class to store a snapshot of environment variables, can be reverted to by
|
||||||
// calling .restore()
|
// calling .restore()
|
||||||
export class EnvironmentSnapshot {
|
export class EnvironmentSnapshot {
|
||||||
|
|
||||||
|
public static push() {
|
||||||
|
this._stack.push(new EnvironmentSnapshot());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static pop() {
|
||||||
|
const snapshot = this._stack.pop();
|
||||||
|
if (!snapshot) {
|
||||||
|
throw new Error("EnvironmentSnapshot stack is empty");
|
||||||
|
}
|
||||||
|
snapshot.restore();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static _stack: EnvironmentSnapshot[] = [];
|
||||||
|
|
||||||
private _oldEnv: NodeJS.ProcessEnv;
|
private _oldEnv: NodeJS.ProcessEnv;
|
||||||
|
|
||||||
public constructor() {
|
public constructor() {
|
||||||
this._oldEnv = clone(process.env);
|
this._oldEnv = clone(process.env);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reset environment variables.
|
// Reset environment variables.
|
||||||
public restore() {
|
public restore() {
|
||||||
Object.assign(process.env, this._oldEnv);
|
Object.assign(process.env, this._oldEnv);
|
||||||
|
Loading…
Reference in New Issue
Block a user