Summary: - Implements MemoryPool for waiting on memory reservations. - Uses MemoryPool to control memory used for stringifying JSON responses in Client.ts - Limits total size of _missedMessages that may be queued for a particular client. - Upgrades ws library, which may reduce memory usage, and allows pausing the websocket for testing. - The upgrade changed subtle behavior corners, requiring various fixes to code and tests. - dos.ts: - Includes Paul's fixes and updates to the dos.ts script for manual stress-testing. - Logging tweaks, to avoid excessive dumps on uncaughtError, and include timestamps. Test Plan: - Includes a test that measures heap size, and fails without memory management. - Includes a unittest for MemoryPool - Some cleanup and additions to TestServer helper; in particular adds makeUserApi() helper used in multiple tests. - Some fixes related to ws upgrade. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3974pull/614/head
Dmitry S
10 months ago
parent
7c114bf600
commit
526a5df157
@ -0,0 +1,114 @@
|
||||
import Deque from 'double-ended-queue';
|
||||
|
||||
/**
|
||||
* Usage:
|
||||
*
|
||||
* OPTION 1, using a callback, which may be async (but doesn't have to be).
|
||||
*
|
||||
* await mpool.withReserved(initialSize, async (updateReservation) => {
|
||||
* ...
|
||||
* updateReservation(newSize); // if needed
|
||||
* ...
|
||||
* });
|
||||
*
|
||||
* OPTION 2, lower-level.
|
||||
*
|
||||
* Note: dispose() MUST be called (e.g. using try/finally). If not called, other work will
|
||||
* eventually deadlock waiting for it.
|
||||
*
|
||||
* const memoryReservation = await mpool.waitAndReserve(initialSize);
|
||||
* try {
|
||||
* ...
|
||||
* memoryReservation.updateReservation(newSize1); // if needed
|
||||
* memoryReservation.updateReservation(newSize2); // if needed
|
||||
* ...
|
||||
* } finally {
|
||||
* memoryReservation.dispose();
|
||||
* }
|
||||
*
|
||||
* With both options, it's common for the initialSize to be a pool estimate. You may call
|
||||
* updateReservation() to update it. If it lowers the estimate, other work may unblock. If it
|
||||
* raises it, it may delay future work, but will have no impact on work that's already unblocked.
|
||||
* So it's always safer for initialSize to be an overestimate.
|
||||
*
|
||||
* When it's hard to estimate initialSize in bytes, you may specify it as e.g.
|
||||
* memPool.getTotalSize() / 20. This way at most 20 such parallel tasks may be unblocked at a
|
||||
* time, and further ones will wait until some release their memory or revise down their estimate.
|
||||
*/
|
||||
export class MemoryPool {
|
||||
private _reservedSize: number = 0;
|
||||
private _queue = new Deque<MemoryAwaiter>();
|
||||
|
||||
constructor(private _totalSize: number) {}
|
||||
|
||||
public getTotalSize(): number { return this._totalSize; }
|
||||
public getReservedSize(): number { return this._reservedSize; }
|
||||
public getAvailableSize(): number { return this._totalSize - this._reservedSize; }
|
||||
public isEmpty(): boolean { return this._reservedSize === 0; }
|
||||
public hasSpace(size: number): boolean { return this._reservedSize + size <= this._totalSize; }
|
||||
|
||||
// To avoid failures, allow reserving more than totalSize when memory pool is empty.
|
||||
public hasSpaceOrIsEmpty(size: number): boolean { return this.hasSpace(size) || this.isEmpty(); }
|
||||
|
||||
public numWaiting(): number { return this._queue.length; }
|
||||
|
||||
public async waitAndReserve(size: number): Promise<MemoryReservation> {
|
||||
if (this.hasSpaceOrIsEmpty(size)) {
|
||||
this._updateReserved(size);
|
||||
} else {
|
||||
await new Promise<void>(resolve => this._queue.push({size, resolve}));
|
||||
}
|
||||
return new MemoryReservation(size, this._updateReserved.bind(this));
|
||||
}
|
||||
|
||||
public async withReserved(size: number, callback: (updateRes: UpdateReservation) => void|Promise<void>) {
|
||||
const memRes = await this.waitAndReserve(size);
|
||||
try {
|
||||
return await callback(memRes.updateReservation.bind(memRes));
|
||||
} finally {
|
||||
memRes.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
// Update the total size. Returns the old size. This is intended for testing.
|
||||
public setTotalSize(newTotalSize: number): number {
|
||||
const oldTotalSize = this._totalSize;
|
||||
this._totalSize = newTotalSize;
|
||||
this._checkWaiting();
|
||||
return oldTotalSize;
|
||||
}
|
||||
|
||||
private _checkWaiting() {
|
||||
while (!this._queue.isEmpty() && this.hasSpaceOrIsEmpty(this._queue.peekFront()!.size)) {
|
||||
const item = this._queue.shift()!;
|
||||
this._updateReserved(item.size);
|
||||
item.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
private _updateReserved(sizeDelta: number): void {
|
||||
this._reservedSize += sizeDelta;
|
||||
this._checkWaiting();
|
||||
}
|
||||
}
|
||||
|
||||
type UpdateReservation = (sizeDelta: number) => void;
|
||||
|
||||
export class MemoryReservation {
|
||||
constructor(private _size: number, private _updateReserved: UpdateReservation) {}
|
||||
|
||||
public updateReservation(newSize: number) {
|
||||
this._updateReserved(newSize - this._size);
|
||||
this._size = newSize;
|
||||
}
|
||||
|
||||
public dispose() {
|
||||
this.updateReservation(0);
|
||||
this._updateReserved = undefined as any; // Make sure we don't keep using it after dispose
|
||||
}
|
||||
}
|
||||
|
||||
interface MemoryAwaiter {
|
||||
size: number;
|
||||
resolve: () => void;
|
||||
}
|
||||
@ -0,0 +1,241 @@
|
||||
import {GristWSConnection} from 'app/client/components/GristWSConnection';
|
||||
import {TableFetchResult} from 'app/common/ActiveDocAPI';
|
||||
import {UserAPIImpl} from 'app/common/UserAPI';
|
||||
import {delay} from 'app/common/delay';
|
||||
import * as log from 'app/server/lib/log';
|
||||
import {getGristConfig} from 'test/gen-server/testUtils';
|
||||
import {prepareDatabase} from 'test/server/lib/helpers/PrepareDatabase';
|
||||
import {TestServer} from 'test/server/lib/helpers/TestServer';
|
||||
import {createTestDir, EnvironmentSnapshot, setTmpLogLevel} from 'test/server/testUtils';
|
||||
import {assert} from 'chai';
|
||||
import * as cookie from 'cookie';
|
||||
import fetch from 'node-fetch';
|
||||
import WebSocket from 'ws';
|
||||
|
||||
describe('ManyFetches', function() {
|
||||
this.timeout(30000);
|
||||
|
||||
setTmpLogLevel('warn'); // Set to 'info' to see what heap size actually is.
|
||||
let oldEnv: EnvironmentSnapshot;
|
||||
|
||||
const userName = 'chimpy';
|
||||
const email = 'chimpy@getgrist.com';
|
||||
const org = 'docs';
|
||||
|
||||
let home: TestServer;
|
||||
let docs: TestServer;
|
||||
let userApi: UserAPIImpl;
|
||||
|
||||
beforeEach(async function() {
|
||||
oldEnv = new EnvironmentSnapshot(); // Needed for prepareDatabase, which changes process.env
|
||||
log.info("Starting servers");
|
||||
const testDir = await createTestDir("ManyFetches");
|
||||
await prepareDatabase(testDir);
|
||||
home = await TestServer.startServer('home', testDir, "home");
|
||||
docs = await TestServer.startServer('docs', testDir, "docs", {
|
||||
// The test verifies memory usage by checking heap sizes. The line below limits doc-worker
|
||||
// process so that it crashes when memory management is wrong. With fetch sizes
|
||||
// in this test, doc-worker's heap size goes from ~90M to ~330M without memory management;
|
||||
// this limit is in the middle as another way to verify that memory management helps.
|
||||
// Without this limit, there is no pressure on node to garbage-collect, so it may use more
|
||||
// memory than we expect, making the test less reliable.
|
||||
NODE_OPTIONS: '--max-old-space-size=210',
|
||||
}, home.serverUrl);
|
||||
userApi = home.makeUserApi(org, userName);
|
||||
});
|
||||
|
||||
afterEach(async function() {
|
||||
// stop all servers
|
||||
await home.stop();
|
||||
await docs.stop();
|
||||
oldEnv.restore();
|
||||
});
|
||||
|
||||
// Assert and log; helpful for working on the test (when setTmpLogLevel is 'info').
|
||||
function assertIsBelow(value: number, expected: number) {
|
||||
log.info("HeapMB", value, `(expected < ${expected})`);
|
||||
assert.isBelow(value, expected);
|
||||
}
|
||||
|
||||
it('should limit the memory used to respond to many simultaneuous fetches', async function() {
|
||||
// Here we create a large document, and fetch it in parallel 200 times, without reading
|
||||
// responses. This test relies on the fact that the server caches the fetched data, so only
|
||||
// the serialized responses to clients are responsible for large memory use. This is the
|
||||
// memory use limited in Client.ts by jsonMemoryPool.
|
||||
|
||||
// Reduce the limit controlling memory for JSON responses from the default of 500MB to 50MB.
|
||||
await docs.testingHooks.commSetClientJsonMemoryLimit(50 * 1024 * 1024);
|
||||
|
||||
// Create a large document where fetches would have a noticeable memory footprint.
|
||||
// 40k rows should produce ~2MB fetch response.
|
||||
const {docId} = await createLargeDoc({rows: 40_000});
|
||||
|
||||
// When we get results, here's a checker that it looks reasonable.
|
||||
function checkResults(results: TableFetchResult[]) {
|
||||
assert.lengthOf(results, 100);
|
||||
for (const res of results) {
|
||||
assert.lengthOf(res.tableData[2], 40_000);
|
||||
assert.lengthOf(res.tableData[3].Num, 40_000);
|
||||
assert.lengthOf(res.tableData[3].Text, 40_000);
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare to make N requests. For N=100, doc-worker should need ~200M of additional memory
|
||||
// without memory management.
|
||||
const N = 100;
|
||||
|
||||
// Helper to get doc-worker's heap size.
|
||||
// If the server dies, testingHooks calls may hang. This wrapper prevents that.
|
||||
const serverErrorPromise = docs.getExitPromise().then(() => { throw new Error("server exited"); });
|
||||
const getMemoryUsage = () => Promise.race([docs.testingHooks.getMemoryUsage(), serverErrorPromise]);
|
||||
const getHeapMB = async () => Math.round((await getMemoryUsage() as NodeJS.MemoryUsage).heapUsed /1024/1024);
|
||||
|
||||
assertIsBelow(await getHeapMB(), 120);
|
||||
|
||||
// Create all the connections, but don't make the fetches just yet.
|
||||
const createConnectionFunc = await prepareGristWSConnection(docId);
|
||||
const connectionsA = Array.from(Array(N), createConnectionFunc);
|
||||
const fetchersA = await Promise.all(connectionsA.map(c => connect(c, docId)));
|
||||
|
||||
const connectionsB = Array.from(Array(N), createConnectionFunc);
|
||||
const fetchersB = await Promise.all(connectionsB.map(c => connect(c, docId)));
|
||||
|
||||
try {
|
||||
assertIsBelow(await getHeapMB(), 120);
|
||||
|
||||
// Start fetches without reading responses. This is a step that should push memory limits.
|
||||
fetchersA.map(f => f.startPausedFetch());
|
||||
|
||||
// Give it a few seconds, enough for server to use what memory it can.
|
||||
await delay(2000);
|
||||
assertIsBelow(await getHeapMB(), 200);
|
||||
|
||||
// Make N more requests. See that memory hasn't spiked.
|
||||
fetchersB.map(f => f.startPausedFetch());
|
||||
await delay(2000);
|
||||
assertIsBelow(await getHeapMB(), 200);
|
||||
|
||||
// Complete the first batch of requests. This allows for the fetches to complete, and for
|
||||
// memory to get released. Also check that results look reasonable.
|
||||
checkResults(await Promise.all(fetchersA.map(f => f.completeFetch())));
|
||||
|
||||
assertIsBelow(await getHeapMB(), 200);
|
||||
|
||||
// Complete the outstanding requests. Memory shouldn't spike.
|
||||
checkResults(await Promise.all(fetchersB.map(f => f.completeFetch())));
|
||||
|
||||
assertIsBelow(await getHeapMB(), 200);
|
||||
|
||||
} finally {
|
||||
fetchersA.map(f => f.end());
|
||||
fetchersB.map(f => f.end());
|
||||
}
|
||||
});
|
||||
|
||||
// Creates a document with the given number of rows, and about 50 bytes per row.
|
||||
async function createLargeDoc({rows}: {rows: number}): Promise<{docId: string}> {
|
||||
log.info("Preparing a doc of %s rows", rows);
|
||||
const ws = (await userApi.getOrgWorkspaces('current'))[0].id;
|
||||
const docId = await userApi.newDoc({name: 'testdoc'}, ws);
|
||||
await userApi.applyUserActions(docId, [['AddTable', 'TestTable', [
|
||||
{id: 'Num', type: 'Numeric'},
|
||||
{id: 'Text', type: 'Text'}
|
||||
]]]);
|
||||
const chunk = 10_000;
|
||||
for (let i = 0; i < rows; i += chunk) {
|
||||
const currentNumRows = Math.min(chunk, rows - i);
|
||||
await userApi.getDocAPI(docId).addRows('TestTable', {
|
||||
// Roughly 8 bytes per row
|
||||
Num: Array.from(Array(currentNumRows), (_, n) => (i + n) * 100),
|
||||
// Roughly 40 bytes per row
|
||||
Text: Array.from(Array(currentNumRows), (_, n) => `Hello, world, again for the ${i + n}th time.`),
|
||||
});
|
||||
}
|
||||
return {docId};
|
||||
}
|
||||
|
||||
// Get all the info for how to create a GristWSConnection, and returns a connection-creating
|
||||
// function.
|
||||
async function prepareGristWSConnection(docId: string): Promise<() => GristWSConnection> {
|
||||
// Use cookies for access to stay as close as possible to regular operation.
|
||||
const resp = await fetch(`${home.serverUrl}/test/session`);
|
||||
const sid = cookie.parse(resp.headers.get('set-cookie')).grist_sid;
|
||||
if (!sid) { throw new Error('no session available'); }
|
||||
await home.testingHooks.setLoginSessionProfile(sid, {name: userName, email}, org);
|
||||
|
||||
// Load the document html.
|
||||
const pageUrl = `${home.serverUrl}/o/docs/doc/${docId}`;
|
||||
const headers = {Cookie: `grist_sid=${sid}`};
|
||||
const doc = await fetch(pageUrl, {headers});
|
||||
const pageBody = await doc.text();
|
||||
|
||||
// Pull out the configuration object embedded in the html.
|
||||
const gristConfig = getGristConfig(pageBody);
|
||||
const {assignmentId, getWorker, homeUrl} = gristConfig;
|
||||
if (!homeUrl) { throw new Error('no homeUrl'); }
|
||||
if (!assignmentId) { throw new Error('no assignmentId'); }
|
||||
const docWorkerUrl = getWorker && getWorker[assignmentId];
|
||||
if (!docWorkerUrl) { throw new Error('no docWorkerUrl'); }
|
||||
|
||||
// Place the config object in window.gristConfig as if we were a
|
||||
// real browser client. GristWSConnection expects to find it there.
|
||||
globalThis.window = globalThis.window || {};
|
||||
(globalThis.window as any).gristConfig = gristConfig;
|
||||
|
||||
// We return a function that constructs a GristWSConnection.
|
||||
return function createConnectionFunc() {
|
||||
let clientId: string = '0';
|
||||
return GristWSConnection.create(null, {
|
||||
makeWebSocket(url: string): any { return new WebSocket(url, undefined, { headers }); },
|
||||
getTimezone() { return Promise.resolve('UTC'); },
|
||||
getPageUrl() { return pageUrl; },
|
||||
getDocWorkerUrl() { return Promise.resolve(docWorkerUrl); },
|
||||
getClientId(did) { return clientId; },
|
||||
getUserSelector() { return ''; },
|
||||
updateClientId(did: string, cid: string) { clientId = cid; },
|
||||
advanceCounter(): string { return '0'; },
|
||||
log(msg, ...args) {},
|
||||
warn(msg, ...args) {},
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
// Actually connect GristWSConnection, open the doc, and return a few methods for next steps.
|
||||
async function connect(connection: GristWSConnection, docId: string) {
|
||||
async function getMessage<T>(eventType: string, filter: (msg: T) => boolean): Promise<T> {
|
||||
return new Promise<T>(resolve => {
|
||||
function callback(msg: T) {
|
||||
if (filter(msg)) { connection.off(eventType, callback); resolve(msg); }
|
||||
}
|
||||
connection.on(eventType, callback);
|
||||
});
|
||||
}
|
||||
|
||||
// Launch the websocket
|
||||
const connectionPromise = getMessage('connectState', (isConnected: boolean) => isConnected);
|
||||
connection.initialize(null);
|
||||
await connectionPromise; // Wait for connection to succeed.
|
||||
|
||||
const openPromise = getMessage('serverMessage', ({reqId}: {reqId?: number}) => (reqId === 0));
|
||||
connection.send(JSON.stringify({reqId: 0, method: 'openDoc', args: [docId]}));
|
||||
await openPromise;
|
||||
|
||||
let fetchPromise: Promise<TableFetchResult>;
|
||||
return {
|
||||
startPausedFetch: () => {
|
||||
fetchPromise = getMessage<any>('serverMessage', ({reqId}: {reqId?: number}) => (reqId === 1));
|
||||
(connection as any)._ws.pause();
|
||||
connection.send(JSON.stringify({reqId: 1, method: 'fetchTable', args: [0, 'TestTable']}));
|
||||
},
|
||||
|
||||
completeFetch: async (): Promise<TableFetchResult> => {
|
||||
(connection as any)._ws.resume();
|
||||
return (await fetchPromise as any).data;
|
||||
},
|
||||
|
||||
end: () => {
|
||||
connection.dispose();
|
||||
},
|
||||
};
|
||||
}
|
||||
});
|
||||
@ -0,0 +1,115 @@
|
||||
import {MemoryPool} from 'app/server/lib/MemoryPool';
|
||||
import {delay} from 'app/common/delay';
|
||||
import {isLongerThan} from 'app/common/gutil';
|
||||
import {assert} from 'chai';
|
||||
import * as sinon from 'sinon';
|
||||
|
||||
async function isResolved(promise: Promise<unknown>): Promise<boolean> {
|
||||
return !await isLongerThan(promise, 0);
|
||||
}
|
||||
|
||||
async function areResolved(...promises: Promise<unknown>[]): Promise<boolean[]> {
|
||||
return Promise.all(promises.map(p => isResolved(p)));
|
||||
}
|
||||
|
||||
function poolInfo(mpool: MemoryPool): {total: number, reserved: number, available: number, awaiters: number} {
|
||||
return {
|
||||
total: mpool.getTotalSize(),
|
||||
reserved: mpool.getReservedSize(),
|
||||
available: mpool.getAvailableSize(),
|
||||
awaiters: mpool.numWaiting(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("MemoryPool", function() {
|
||||
|
||||
afterEach(() => {
|
||||
sinon.restore();
|
||||
});
|
||||
|
||||
it("should wait for enough space", async function() {
|
||||
const mpool = new MemoryPool(1000);
|
||||
const spy = sinon.spy();
|
||||
let r1: () => void;
|
||||
let r2: () => void;
|
||||
let r3: () => void;
|
||||
let r4: () => void;
|
||||
const w1 = new Promise<void>(r => { r1 = r; });
|
||||
const w2 = new Promise<void>(r => { r2 = r; });
|
||||
const w3 = new Promise<void>(r => { r3 = r; });
|
||||
const w4 = new Promise<void>(r => { r4 = r; });
|
||||
const p1 = mpool.withReserved(400, () => { spy(1); return w1; });
|
||||
const p2 = mpool.withReserved(400, () => { spy(2); return w2; });
|
||||
const p3 = mpool.withReserved(400, () => { spy(3); return w3; });
|
||||
const p4 = mpool.withReserved(400, () => { spy(4); return w4; });
|
||||
|
||||
// Only two callbacks run initially.
|
||||
await delay(10);
|
||||
assert.deepEqual(spy.args, [[1], [2]]);
|
||||
|
||||
// Others are waiting for something to finish.
|
||||
await delay(50);
|
||||
assert.deepEqual(spy.args, [[1], [2]]);
|
||||
|
||||
// Once 2nd task finishes, the next one should run.
|
||||
r2!();
|
||||
await delay(10);
|
||||
assert.deepEqual(spy.args, [[1], [2], [3]]);
|
||||
await delay(50);
|
||||
assert.deepEqual(spy.args, [[1], [2], [3]]);
|
||||
|
||||
// Once another task finishes, the last one should run.
|
||||
r3!();
|
||||
await delay(10);
|
||||
assert.deepEqual(spy.args, [[1], [2], [3], [4]]);
|
||||
|
||||
// Let all tasks finish.
|
||||
r1!();
|
||||
r4!();
|
||||
await delay(10);
|
||||
assert.deepEqual(spy.args, [[1], [2], [3], [4]]);
|
||||
await Promise.all([p1, p2, p3, p4]);
|
||||
});
|
||||
|
||||
it("should allow adjusting reservation", async function() {
|
||||
const mpool = new MemoryPool(1000);
|
||||
const res1p = mpool.waitAndReserve(600);
|
||||
const res2p = mpool.waitAndReserve(600);
|
||||
|
||||
// Initially only the first reservation can happen.
|
||||
assert.deepEqual(poolInfo(mpool), {total: 1000, reserved: 600, available: 400, awaiters: 1});
|
||||
assert.deepEqual(await areResolved(res1p, res2p), [true, false]);
|
||||
|
||||
// Once the first reservation is adjusted, the next one should go.
|
||||
const res1 = await res1p;
|
||||
res1.updateReservation(400);
|
||||
assert.deepEqual(poolInfo(mpool), {total: 1000, reserved: 1000, available: 0, awaiters: 0});
|
||||
assert.deepEqual(await areResolved(res1p, res2p), [true, true]);
|
||||
|
||||
const res2 = await res2p;
|
||||
|
||||
// Try some more complex combinations.
|
||||
const res3p = mpool.waitAndReserve(200);
|
||||
const res4p = mpool.waitAndReserve(200);
|
||||
const res5p = mpool.waitAndReserve(200);
|
||||
assert.deepEqual(poolInfo(mpool), {total: 1000, reserved: 1000, available: 0, awaiters: 3});
|
||||
assert.deepEqual(await areResolved(res3p, res4p, res5p), [false, false, false]);
|
||||
|
||||
res1.updateReservation(100); // 300 units freed.
|
||||
assert.deepEqual(poolInfo(mpool), {total: 1000, reserved: 900, available: 100, awaiters: 2});
|
||||
assert.deepEqual(await areResolved(res3p, res4p, res5p), [true, false, false]);
|
||||
|
||||
res1.dispose(); // Another 100 freed.
|
||||
assert.deepEqual(poolInfo(mpool), {total: 1000, reserved: 1000, available: 0, awaiters: 1});
|
||||
assert.deepEqual(await areResolved(res3p, res4p, res5p), [true, true, false]);
|
||||
|
||||
res2.dispose(); // Lots freed.
|
||||
assert.deepEqual(poolInfo(mpool), {total: 1000, reserved: 600, available: 400, awaiters: 0});
|
||||
assert.deepEqual(await areResolved(res3p, res4p, res5p), [true, true, true]);
|
||||
|
||||
(await res5p).dispose();
|
||||
(await res4p).dispose();
|
||||
(await res3p).dispose();
|
||||
assert.deepEqual(poolInfo(mpool), {total: 1000, reserved: 0, available: 1000, awaiters: 0});
|
||||
});
|
||||
});
|
||||
Loading…
Reference in new issue