Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

(core) updates from grist-core

Browse Source
This commit is contained in:
Paul Fitzpatrick 2024-04-01 09:22:25 -04:00
commit 4736ca490c
24 changed files with 1006 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

152
app/client/components/GristClientSocket.ts Normal file
View File

@ -0,0 +1,152 @@
import WS from 'ws';
import { Socket as EIOSocket } from 'engine.io-client';
export interface GristClientSocketOptions {
headers?: Record<string, string>;
}
export class GristClientSocket {
// Exactly one of _wsSocket and _eioSocket will be set at any one time.
private _wsSocket: WS.WebSocket | WebSocket | undefined;
private _eioSocket: EIOSocket | undefined;
// Set to true if a WebSocket connection (in _wsSocket) was succesfully
// established. Errors from the underlying WebSocket are not forwarded to
// the client until that point, in case we end up downgrading to Engine.IO.
private _wsConnected: boolean = false;
private _messageHandler: null | ((data: string) => void);
private _openHandler: null | (() => void);
private _errorHandler: null | ((err: Error) => void);
private _closeHandler: null | (() => void);
constructor(private _url: string, private _options?: GristClientSocketOptions) {
this._createWSSocket();
}
public set onmessage(cb: null | ((data: string) => void)) {
this._messageHandler = cb;
}
public set onopen(cb: null | (() => void)) {
this._openHandler = cb;
}
public set onerror(cb: null | ((err: Error) => void)) {
this._errorHandler = cb;
}
public set onclose(cb: null | (() => void)) {
this._closeHandler = cb;
}
public close() {
if (this._wsSocket) {
this._wsSocket.close();
} else {
this._eioSocket!.close();
}
}
public send(data: string) {
if (this._wsSocket) {
this._wsSocket.send(data);
} else {
this._eioSocket!.send(data);
}
}
// pause() and resume() are used for tests and assume a WS.WebSocket transport
public pause() {
(this._wsSocket as WS.WebSocket)?.pause();
}
public resume() {
(this._wsSocket as WS.WebSocket)?.resume();
}
private _createWSSocket() {
if (typeof WebSocket !== 'undefined') {
this._wsSocket = new WebSocket(this._url);
} else {
this._wsSocket = new WS(this._url, undefined, this._options);
}
this._wsSocket.onmessage = this._onWSMessage.bind(this);
this._wsSocket.onopen = this._onWSOpen.bind(this);
this._wsSocket.onerror = this._onWSError.bind(this);
this._wsSocket.onclose = this._onWSClose.bind(this);
}
private _destroyWSSocket() {
if (this._wsSocket) {
this._wsSocket.onmessage = null;
this._wsSocket.onopen = null;
this._wsSocket.onerror = null;
this._wsSocket.onclose = null;
this._wsSocket = undefined;
}
}
private _onWSMessage(event: WS.MessageEvent | MessageEvent<any>) {
// event.data is guaranteed to be a string here because we only send text frames.
// https://developer.mozilla.org/en-US/docs/Web/API/WebSocket/message_event#event_properties
this._messageHandler?.(event.data);
}
private _onWSOpen() {
// The connection was established successfully. Any future events can now
// be forwarded to the client.
this._wsConnected = true;
this._openHandler?.();
}
private _onWSError(ev: Event) {
if (!this._wsConnected) {
// The WebSocket connection attempt failed. Switch to Engine.IO.
this._destroyWSSocket();
this._createEIOSocket();
return;
}
// WebSocket error events are deliberately void of information,
// see https://websockets.spec.whatwg.org/#eventdef-websocket-error,
// so we ignore the incoming event.
this._errorHandler?.(new Error("WebSocket error"));
}
private _onWSClose() {
this._closeHandler?.();
}
private _createEIOSocket() {
this._eioSocket = new EIOSocket(this._url, {
path: new URL(this._url).pathname,
addTrailingSlash: false,
transports: ['polling'],
upgrade: false,
extraHeaders: this._options?.headers,
withCredentials: true,
});
this._eioSocket.on('message', this._onEIOMessage.bind(this));
this._eioSocket.on('open', this._onEIOOpen.bind(this));
this._eioSocket.on('error', this._onEIOError.bind(this));
this._eioSocket.on('close', this._onEIOClose.bind(this));
}
private _onEIOMessage(data: string) {
this._messageHandler?.(data);
}
private _onEIOOpen() {
this._openHandler?.();
}
private _onEIOError(err: string | Error) {
this._errorHandler?.(typeof err === "string" ? new Error(err) : err);
}
private _onEIOClose() {
this._closeHandler?.();
}
}

20
app/client/components/GristWSConnection.ts
View File

@ -9,6 +9,7 @@ import {addOrgToPath, docUrl, getGristConfig} from 'app/common/urlUtils';
import {UserAPI} from 'app/common/UserAPI';
import {Events as BackboneEvents} from 'backbone';
import {Disposable} from 'grainjs';
import {GristClientSocket} from './GristClientSocket';
const G = getBrowserGlobals('window');
const reconnectInterval = [1000, 1000, 2000, 5000, 10000];
@ -37,7 +38,7 @@ async function getDocWorkerUrl(assignmentId: string|null): Promise<string|null>
export interface GristWSSettings {
// A factory function for creating the WebSocket so that we can use from node
// or browser.
makeWebSocket(url: string): WebSocket;
makeWebSocket(url: string): GristClientSocket;
// A function for getting the timezone, so the code can be used outside webpack -
// currently a timezone library is lazy loaded in a way that doesn't quite work
@ -74,7 +75,7 @@ export interface GristWSSettings {
export class GristWSSettingsBrowser implements GristWSSettings {
private _sessionStorage = getSessionStorage();
public makeWebSocket(url: string) { return new WebSocket(url); }
public makeWebSocket(url: string) { return new GristClientSocket(url); }
public getTimezone() { return guessTimezone(); }
public getPageUrl() { return G.window.location.href; }
public async getDocWorkerUrl(assignmentId: string|null) {
@ -125,7 +126,7 @@ export class GristWSConnection extends Disposable {
private _reconnectTimeout: ReturnType<typeof setTimeout> | null = null;
private _reconnectAttempts: number = 0;
private _wantReconnect: boolean = true;
private _ws: WebSocket|null = null;
private _ws: GristClientSocket|null = null;
// The server sends incremental seqId numbers with each message on the connection, starting with
// 0. We keep track of them to allow for seamless reconnects.
@ -211,17 +212,16 @@ export class GristWSConnection extends Disposable {
}
/**
* @event serverMessage Triggered when a message arrives from the server. Callbacks receive
* the raw message data as an additional argument.
* Triggered when a message arrives from the server.
*/
public onmessage(ev: MessageEvent) {
public onmessage(data: string) {
if (!this._ws) {
// It's possible to receive a message after we disconnect, at least in tests (where
// WebSocket is a node library). Ignoring is easier than unsubscribing properly.
return;
}
this._scheduleHeartbeat();
this._processReceivedMessage(ev.data, true);
this._processReceivedMessage(data, true);
}
public send(message: any) {
@ -317,7 +317,7 @@ export class GristWSConnection extends Disposable {
private _sendHeartbeat() {
this.send(JSON.stringify({
beat: 'alive',
url: G.window.location.href,
url: this._settings.getPageUrl(),
docId: this._assignmentId,
}));
}
@ -352,8 +352,8 @@ export class GristWSConnection extends Disposable {
this._ws.onmessage = this.onmessage.bind(this);
this._ws.onerror = (ev: Event|ErrorEvent) => {
this._log('GristWSConnection: onerror', 'error' in ev ? String(ev.error) : ev);
this._ws.onerror = (err: Error) => {
this._log('GristWSConnection: onerror', String(err));
};
this._ws.onclose = () => {

36
app/server/lib/Client.ts
View File

@ -22,7 +22,7 @@ import {fromCallback} from 'app/server/lib/serverUtils';
import {i18n} from 'i18next';
import * as crypto from 'crypto';
import moment from 'moment';
import * as WebSocket from 'ws';
import {GristServerSocket} from 'app/server/lib/GristServerSocket';
// How many messages and bytes to accumulate for a disconnected client before booting it.
// The benefit is that a client who temporarily disconnects and reconnects without missing much,
@ -97,8 +97,7 @@ export class Client {
private _missedMessagesTotalLength: number = 0;
private _destroyTimer: NodeJS.Timer|null = null;
private _destroyed: boolean = false;
private _websocket: WebSocket|null;
private _websocketEventHandlers: Array<{event: string, handler: (...args: any[]) => void}> = [];
private _websocket: GristServerSocket|null;
private _org: string|null = null;
private _profile: UserProfile|null = null;
private _user: FullUser|undefined = undefined;
@ -131,18 +130,14 @@ export class Client {
return this._locale;
}
public setConnection(websocket: WebSocket, counter: string|null, browserSettings: BrowserSettings) {
public setConnection(websocket: GristServerSocket, counter: string|null, browserSettings: BrowserSettings) {
this._websocket = websocket;
this._counter = counter;
this.browserSettings = browserSettings;
const addHandler = (event: string, handler: (...args: any[]) => void) => {
websocket.on(event, handler);
this._websocketEventHandlers.push({event, handler});
};
addHandler('error', (err: unknown) => this._onError(err));
addHandler('close', () => this._onClose());
addHandler('message', (msg: string) => this._onMessage(msg));
websocket.onerror = (err: Error) => this._onError(err);
websocket.onclose = () => this._onClose();
websocket.onmessage = (msg: string) => this._onMessage(msg);
}
/**
@ -189,7 +184,7 @@ export class Client {
public interruptConnection() {
if (this._websocket) {
this._removeWebsocketListeners();
this._websocket.removeAllListeners();
this._websocket.terminate(); // close() is inadequate when ws routed via loadbalancer
this._websocket = null;
}
@ -359,7 +354,7 @@ export class Client {
// See also my report at https://stackoverflow.com/a/48411315/328565
await delay(250);
if (!this._destroyed && this._websocket?.readyState === WebSocket.OPEN) {
if (!this._destroyed && this._websocket?.isOpen) {
await this._sendToWebsocket(JSON.stringify({...clientConnectMsg, dup: true}));
}
} catch (err) {
@ -604,7 +599,7 @@ export class Client {
/**
* Processes an error on the websocket.
*/
private _onError(err: unknown) {
private _onError(err: Error) {
this._log.warn(null, "onError", err);
// TODO Make sure that this is followed by onClose when the connection is lost.
}
@ -613,7 +608,7 @@ export class Client {
* Processes the closing of a websocket.
*/
private _onClose() {
this._removeWebsocketListeners();
this._websocket?.removeAllListeners();
// Remove all references to the websocket.
this._websocket = null;
@ -629,15 +624,4 @@ export class Client {
this._destroyTimer = setTimeout(() => this.destroy(), Deps.clientRemovalTimeoutMs);
}
}
private _removeWebsocketListeners() {
if (this._websocket) {
// Avoiding websocket.removeAllListeners() because WebSocket.Server registers listeners
// internally for websockets it keeps track of, and we should not accidentally remove those.
for (const {event, handler} of this._websocketEventHandlers) {
this._websocket.off(event, handler);
}
this._websocketEventHandlers = [];
}
}
}

39
app/server/lib/Comm.ts
View File

@ -35,7 +35,8 @@
import {EventEmitter} from 'events';
import * as http from 'http';
import * as https from 'https';
import * as WebSocket from 'ws';
import {GristSocketServer} from 'app/server/lib/GristSocketServer';
import {GristServerSocket} from 'app/server/lib/GristServerSocket';
import {parseFirstUrlPart} from 'app/common/gristUrls';
import {firstDefined, safeJsonParse} from 'app/common/gutil';
@ -50,6 +51,7 @@ import {localeFromRequest} from 'app/server/lib/ServerLocale';
import {fromCallback} from 'app/server/lib/serverUtils';
import {Sessions} from 'app/server/lib/Sessions';
import {i18n} from 'i18next';
import { trustOrigin } from './requestUtils';
export interface CommOptions {
sessions: Sessions; // A collection of all sessions for this instance of Grist
@ -74,7 +76,7 @@ export interface CommOptions {
export class Comm extends EventEmitter {
// Collection of all sessions; maps sessionIds to ScopedSession objects.
public readonly sessions: Sessions = this._options.sessions;
private _wss: WebSocket.Server[]|null = null;
private _wss: GristSocketServer[]|null = null;
private _clients = new Map<string, Client>(); // Maps clientIds to Client objects.
@ -146,11 +148,6 @@ export class Comm extends EventEmitter {
public async testServerShutdown() {
if (this._wss) {
for (const wssi of this._wss) {
// Terminate all clients. WebSocket.Server used to do it automatically in close() but no
// longer does (see https://github.com/websockets/ws/pull/1904#discussion_r668844565).
for (const ws of wssi.clients) {
ws.terminate();
}
await fromCallback((cb) => wssi.close(cb));
}
this._wss = null;
@ -204,14 +201,7 @@ export class Comm extends EventEmitter {
/**
* Processes a new websocket connection, and associates the websocket and a Client object.
*/
private async _onWebSocketConnection(websocket: WebSocket, req: http.IncomingMessage) {
if (this._options.hosts) {
// DocWorker ID (/dw/) and version tag (/v/) may be present in this request but are not
// needed. addOrgInfo assumes req.url starts with /o/ if present.
req.url = parseFirstUrlPart('dw', req.url!).path;
req.url = parseFirstUrlPart('v', req.url).path;
await this._options.hosts.addOrgInfo(req);
}
private async _onWebSocketConnection(websocket: GristServerSocket, req: http.IncomingMessage) {
// Parse the cookie in the request to get the sessionId.
const sessionId = this.sessions.getSessionIdFromRequest(req);
@ -255,15 +245,28 @@ export class Comm extends EventEmitter {
if (this._options.httpsServer) { servers.push(this._options.httpsServer); }
const wss = [];
for (const server of servers) {
const wssi = new WebSocket.Server({server});
wssi.on('connection', async (websocket: WebSocket, req) => {
const wssi = new GristSocketServer(server, {
verifyClient: async (req: http.IncomingMessage) => {
if (this._options.hosts) {
// DocWorker ID (/dw/) and version tag (/v/) may be present in this request but are not
// needed. addOrgInfo assumes req.url starts with /o/ if present.
req.url = parseFirstUrlPart('dw', req.url!).path;
req.url = parseFirstUrlPart('v', req.url).path;
await this._options.hosts.addOrgInfo(req);
}
return trustOrigin(req);
}
});
wssi.onconnection = async (websocket: GristServerSocket, req) => {
try {
await this._onWebSocketConnection(websocket, req);
} catch (e) {
log.error("Comm connection for %s threw exception: %s", req.url, e.message);
websocket.terminate(); // close() is inadequate when ws routed via loadbalancer
}
});
};
wss.push(wssi);
}
return wss;

138
app/server/lib/GristServerSocket.ts Normal file
View File

@ -0,0 +1,138 @@
import * as WS from 'ws';
import * as EIO from 'engine.io';
export abstract class GristServerSocket {
public abstract set onerror(handler: (err: Error) => void);
public abstract set onclose(handler: () => void);
public abstract set onmessage(handler: (data: string) => void);
public abstract removeAllListeners(): void;
public abstract close(): void;
public abstract terminate(): void;
public abstract get isOpen(): boolean;
public abstract send(data: string, cb?: (err?: Error) => void): void;
}
export class GristServerSocketEIO extends GristServerSocket {
private _eventHandlers: Array<{ event: string, handler: (...args: any[]) => void }> = [];
private _messageCounter = 0;
// Engine.IO only invokes send() callbacks on success. We keep a map of
// send callbacks for messages in flight so that we can invoke them for
// any messages still unsent upon receiving a "close" event.
private _messageCallbacks: Map<number, (err: Error) => void> = new Map();
constructor(private _socket: EIO.Socket) { super(); }
public set onerror(handler: (err: Error) => void) {
// Note that as far as I can tell, Engine.IO sockets never emit "error"
// but instead include error information in the "close" event.
this._socket.on('error', handler);
this._eventHandlers.push({ event: 'error', handler });
}
public set onclose(handler: () => void) {
const wrappedHandler = (reason: string, description: any) => {
// In practice, when available, description has more error details,
// possibly in the form of an Error object.
const maybeErr = description ?? reason;
const err = maybeErr instanceof Error ? maybeErr : new Error(maybeErr);
for (const cb of this._messageCallbacks.values()) {
cb(err);
}
this._messageCallbacks.clear();
handler();
};
this._socket.on('close', wrappedHandler);
this._eventHandlers.push({ event: 'close', handler: wrappedHandler });
}
public set onmessage(handler: (data: string) => void) {
const wrappedHandler = (msg: Buffer) => {
handler(msg.toString());
};
this._socket.on('message', wrappedHandler);
this._eventHandlers.push({ event: 'message', handler: wrappedHandler });
}
public removeAllListeners() {
for (const { event, handler } of this._eventHandlers) {
this._socket.off(event, handler);
}
this._eventHandlers = [];
}
public close() {
this._socket.close();
}
// Terminates the connection without waiting for the client to close its own side.
public terminate() {
// Trigger a normal close. For the polling transport, this is sufficient and instantaneous.
this._socket.close(/* discard */ true);
}
public get isOpen() {
return this._socket.readyState === 'open';
}
public send(data: string, cb?: (err?: Error) => void) {
const msgNum = this._messageCounter++;
if (cb) {
this._messageCallbacks.set(msgNum, cb);
}
this._socket.send(data, {}, () => {
if (cb && this._messageCallbacks.delete(msgNum)) {
// send was successful: pass no Error to callback
cb();
}
});
}
}
export class GristServerSocketWS extends GristServerSocket {
private _eventHandlers: Array<{ event: string, handler: (...args: any[]) => void }> = [];
constructor(private _ws: WS.WebSocket) { super(); }
public set onerror(handler: (err: Error) => void) {
this._ws.on('error', handler);
this._eventHandlers.push({ event: 'error', handler });
}
public set onclose(handler: () => void) {
this._ws.on('close', handler);
this._eventHandlers.push({ event: 'close', handler });
}
public set onmessage(handler: (data: string) => void) {
const wrappedHandler = (msg: Buffer) => handler(msg.toString());
this._ws.on('message', wrappedHandler);
this._eventHandlers.push({ event: 'message', handler: wrappedHandler });
}
public removeAllListeners() {
// Avoiding websocket.removeAllListeners() because WS.Server registers listeners
// internally for websockets it keeps track of, and we should not accidentally remove those.
for (const { event, handler } of this._eventHandlers) {
this._ws.off(event, handler);
}
this._eventHandlers = [];
}
public close() {
this._ws.close();
}
public terminate() {
this._ws.terminate();
}
public get isOpen() {
return this._ws.readyState === WS.OPEN;
}
public send(data: string, cb?: (err?: Error) => void) {
this._ws.send(data, cb);
}
}

111
app/server/lib/GristSocketServer.ts Normal file
View File

@ -0,0 +1,111 @@
import * as http from 'http';
import * as WS from 'ws';
import * as EIO from 'engine.io';
import {GristServerSocket, GristServerSocketEIO, GristServerSocketWS} from './GristServerSocket';
import * as net from 'net';
const MAX_PAYLOAD = 100e6;
export interface GristSocketServerOptions {
verifyClient?: (request: http.IncomingMessage) => Promise<boolean>;
}
export class GristSocketServer {
private _wsServer: WS.Server;
private _eioServer: EIO.Server;
private _connectionHandler: (socket: GristServerSocket, req: http.IncomingMessage) => void;
constructor(server: http.Server, private _options?: GristSocketServerOptions) {
this._wsServer = new WS.Server({ noServer: true, maxPayload: MAX_PAYLOAD });
this._eioServer = new EIO.Server({
// We only use Engine.IO for its polling transport,
// so we disable the built-in Engine.IO upgrade mechanism.
allowUpgrades: false,
transports: ['polling'],
maxHttpBufferSize: MAX_PAYLOAD,
cors: {
// This will cause Engine.IO to reflect any client-provided Origin into
// the Access-Control-Allow-Origin header, essentially disabling the
// protection offered by the Same-Origin Policy. This sounds insecure
// but is actually the security model of native WebSockets (they are
// not covered by SOP; any webpage can open a WebSocket connecting to
// any other domain, including the target domain's cookies; it is up to
// the receiving server to check the request's Origin header). Since
// the connection attempt is validated in `verifyClient` later,
// it is safe to let any client attempt a connection here.
origin: true,
// We need to allow the client to send its cookies. See above for the
// reasoning on why it is safe to do so.
credentials: true,
methods: ["GET", "POST"],
},
});
this._eioServer.on('connection', this._onEIOConnection.bind(this));
this._attach(server);
}
public set onconnection(handler: (socket: GristServerSocket, req: http.IncomingMessage) => void) {
this._connectionHandler = handler;
}
public close(cb: (...args: any[]) => void) {
this._eioServer.close();
// Terminate all clients. WS.Server used to do it automatically in close() but no
// longer does (see https://github.com/websockets/ws/pull/1904#discussion_r668844565).
for (const ws of this._wsServer.clients) {
ws.terminate();
}
this._wsServer.close(cb);
}
private _attach(server: http.Server) {
// Forward all WebSocket upgrade requests to WS
server.on('upgrade', async (request, socket, head) => {
if (this._options?.verifyClient && !await this._options.verifyClient(request)) {
// Because we are handling an "upgrade" event, we don't have access to
// a "response" object, just the raw socket. We can still construct
// a well-formed HTTP error response.
socket.write('HTTP/1.1 403 Forbidden\r\n\r\n');
socket.destroy();
return;
}
this._wsServer.handleUpgrade(request, socket as net.Socket, head, (client) => {
this._connectionHandler?.(new GristServerSocketWS(client), request);
});
});
// At this point an Express app is installed as the handler for the server's
// "request" event. We need to install our own listener instead, to intercept
// requests that are meant for the Engine.IO polling implementation.
const listeners = [...server.listeners("request")];
server.removeAllListeners("request");
server.on("request", async (req, res) => {
// Intercept requests that have transport=polling in their querystring
if (/[&?]transport=polling(&|$)/.test(req.url ?? '')) {
if (this._options?.verifyClient && !await this._options.verifyClient(req)) {
res.writeHead(403).end();
return;
}
this._eioServer.handleRequest(req, res);
} else {
// Otherwise fallback to the pre-existing listener(s)
for (const listener of listeners) {
listener.call(server, req, res);
}
}
});
server.on("close", this.close.bind(this));
}
private _onEIOConnection(socket: EIO.Socket) {
const req = socket.request;
(socket as any).request = null; // Free initial request as recommended in the Engine.IO documentation
this._connectionHandler?.(new GristServerSocketEIO(socket), req);
}
}

32
app/server/lib/requestUtils.ts
View File

@ -8,7 +8,9 @@ import {RequestWithGrist} from 'app/server/lib/GristServer';
import log from 'app/server/lib/log';
import {Permit} from 'app/server/lib/Permit';
import {Request, Response} from 'express';
import { IncomingMessage } from 'http';
import {Writable} from 'stream';
import { TLSSocket } from 'tls';
// log api details outside of dev environment (when GRIST_HOSTED_VERSION is set)
const shouldLogApiDetails = Boolean(process.env.GRIST_HOSTED_VERSION);
@ -75,33 +77,33 @@ export function getOrgUrl(req: Request, path: string = '/') {
}
/**
* Returns true for requests from permitted origins. For such requests, an
* "Access-Control-Allow-Origin" header is added to the response. Vary: Origin
* is also set to reflect the fact that the headers are a function of the origin,
* to prevent inappropriate caching on the browser's side.
* Returns true for requests from permitted origins. For such requests, if
* a Response object is provided, an "Access-Control-Allow-Origin" header is added
* to the response. Vary: Origin is also set to reflect the fact that the headers
* are a function of the origin, to prevent inappropriate caching on the browser's side.
*/
export function trustOrigin(req: Request, resp: Response): boolean {
export function trustOrigin(req: IncomingMessage, resp?: Response): boolean {
// TODO: We may want to consider changing allowed origin values in the future.
// Note that the request origin is undefined for non-CORS requests.
const origin = req.get('origin');
const origin = req.headers.origin;
if (!origin) { return true; } // Not a CORS request.
if (process.env.GRIST_HOST && req.hostname === process.env.GRIST_HOST) { return true; }
if (!allowHost(req, new URL(origin))) { return false; }
if (resp) {
// For a request to a custom domain, the full hostname must match.
resp.header("Access-Control-Allow-Origin", origin);
resp.header("Vary", "Origin");
}
return true;
}
// Returns whether req satisfies the given allowedHost. Unless req is to a custom domain, it is
// enough if only the base domains match. Differing ports are allowed, which helps in dev/testing.
export function allowHost(req: Request, allowedHost: string|URL) {
const mreq = req as RequestWithOrg;
export function allowHost(req: IncomingMessage, allowedHost: string|URL) {
const proto = getEndUserProtocol(req);
const actualUrl = new URL(getOriginUrl(req));
const allowedUrl = (typeof allowedHost === 'string') ? new URL(`${proto}://${allowedHost}`) : allowedHost;
if (mreq.isCustomHost) {
if ((req as RequestWithOrg).isCustomHost) {
// For a request to a custom domain, the full hostname must match.
return actualUrl.hostname === allowedUrl.hostname;
} else {
@ -330,8 +332,8 @@ export interface RequestWithGristInfo extends Request {
* https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto
* https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/x-forwarded-headers.html
*/
export function getOriginUrl(req: Request) {
const host = req.get('host')!;
export function getOriginUrl(req: IncomingMessage) {
const host = req.headers.host;
const protocol = getEndUserProtocol(req);
return `${protocol}://${host}`;
}
@ -342,11 +344,13 @@ export function getOriginUrl(req: Request) {
* otherwise X-Forwarded-Proto is set on the provided request, otherwise
* the protocol of the request itself.
*/
export function getEndUserProtocol(req: Request) {
export function getEndUserProtocol(req: IncomingMessage) {
if (process.env.APP_HOME_URL) {
return new URL(process.env.APP_HOME_URL).protocol.replace(':', '');
}
return req.get("X-Forwarded-Proto") || req.protocol;
// TODO we shouldn't blindly trust X-Forwarded-Proto. See the Express approach:
// https://expressjs.com/en/5x/api.html#trust.proxy.options.table
return req.headers["x-forwarded-proto"] || ((req.socket as TLSSocket).encrypted ? 'https' : 'http');
}
/**

2
buildtools/update_schema.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Regenerates typescript files with schema and sql for grist documents.
# This needs to run whenever the document schema is changed in the data

2
buildtools/update_type_info.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e

4
package.json
View File

@ -82,7 +82,7 @@
"@types/tmp": "0.0.33",
"@types/uuid": "3.4.4",
"@types/which": "2.0.1",
"@types/ws": "^6",
"@types/ws": "^8",
"@typescript-eslint/eslint-plugin": "5.29.0",
"@typescript-eslint/parser": "5.29.0",
"app-module-path": "2.2.0",
@ -139,6 +139,8 @@
"diff-match-patch": "1.0.5",
"dompurify": "3.0.6",
"double-ended-queue": "2.1.0-0",
"engine.io": "^6.5.4",
"engine.io-client": "^6.5.3",
"exceljs": "4.2.1",
"express": "4.18.2",
"file-type": "16.5.4",

2
sandbox/bundle_as_wheel.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Package up Grist code as a stand-alone wheel.
# This is useful for grist-static.

2
sandbox/pyodide/build_packages.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e

2
sandbox/pyodide/setup.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e

39
static/locales/en.client.json
View File

@ -655,7 +655,9 @@
"Submit button label": "Submit button label",
"Success text": "Success text",
"Table column name": "Table column name",
"Enter redirect URL": "Enter redirect URL"
"Enter redirect URL": "Enter redirect URL",
"No field selected": "No field selected",
"Select a field in the form widget to configure.": "Select a field in the form widget to configure."
},
"RowContextMenu": {
"Copy anchor link": "Copy anchor link",
@ -692,7 +694,12 @@
"Unsaved": "Unsaved",
"Work on a Copy": "Work on a Copy",
"Share": "Share",
"Download...": "Download..."
"Download...": "Download...",
"Comma Separated Values (.csv)": "Comma Separated Values (.csv)",
"DOO Separated Values (.dsv)": "DOO Separated Values (.dsv)",
"Export as...": "Export as...",
"Microsoft Excel (.xlsx)": "Microsoft Excel (.xlsx)",
"Tab Separated Values (.tsv)": "Tab Separated Values (.tsv)"
},
"SiteSwitcher": {
"Create new team site": "Create new team site",
@ -1325,7 +1332,21 @@
"Publish": "Publish",
"Publish your form?": "Publish your form?",
"Unpublish": "Unpublish",
"Unpublish your form?": "Unpublish your form?"
"Unpublish your form?": "Unpublish your form?",
"Anyone with the link below can see the empty form and submit a response.": "Anyone with the link below can see the empty form and submit a response.",
"Are you sure you want to reset your form?": "Are you sure you want to reset your form?",
"Code copied to clipboard": "Code copied to clipboard",
"Copy code": "Copy code",
"Copy link": "Copy link",
"Embed this form": "Embed this form",
"Link copied to clipboard": "Link copied to clipboard",
"Preview": "Preview",
"Reset": "Reset",
"Reset form": "Reset form",
"Save your document to publish this form.": "Save your document to publish this form.",
"Share": "Share",
"Share this form": "Share this form",
"View": "View"
},
"Editor": {
"Delete": "Delete"
@ -1388,5 +1409,17 @@
"This week": "This week",
"This year": "This year",
"Today": "Today"
},
"MappedFieldsConfig": {
"Clear": "Clear",
"Map fields": "Map fields",
"Mapped": "Mapped",
"Select All": "Select All",
"Unmap fields": "Unmap fields",
"Unmapped": "Unmapped"
},
"Section": {
"Insert section above": "Insert section above",
"Insert section below": "Insert section below"
}
}

39
static/locales/es.client.json
View File

@ -574,7 +574,9 @@
"Redirection": "Redirección",
"Required field": "Campo requerido",
"Hidden field": "Campo oculto",
"Redirect automatically after submission": "Redirigir automáticamente después del envío"
"Redirect automatically after submission": "Redirigir automáticamente después del envío",
"No field selected": "Ningún campo seleccionado",
"Select a field in the form widget to configure.": "Seleccione un campo del widget del formulario para configurarlo."
},
"RowContextMenu": {
"Copy anchor link": "Copiar enlace de anclaje",
@ -610,7 +612,12 @@
"Unsaved": "No guardado",
"Work on a Copy": "Trabajar en una copia",
"Share": "Compartir",
"Download...": "Descargar..."
"Download...": "Descargar...",
"Comma Separated Values (.csv)": "Valores separados por comas (.csv)",
"DOO Separated Values (.dsv)": "Valores separados DOO (.dsv)",
"Tab Separated Values (.tsv)": "Valores separados por tabulaciones (.tsv)",
"Export as...": "Exportar como...",
"Microsoft Excel (.xlsx)": "Microsoft Excel (.xlsx)"
},
"SiteSwitcher": {
"Create new team site": "Crear nuevo sitio de equipo",
@ -1396,7 +1403,21 @@
"Publish": "Publicar",
"Publish your form?": "¿Publicar su formulario?",
"Unpublish your form?": "¿Anular la publicación de su formulario?",
"Unpublish": "Anular la publicación"
"Unpublish": "Anular la publicación",
"Are you sure you want to reset your form?": "¿Estás seguro de que quieres restablecer tu formulario?",
"Copy code": "Copiar mi código",
"Embed this form": "Insertar este formulario",
"Reset": "Restablecer",
"Share": "Compartir",
"Copy link": "Copiar enlace",
"Anyone with the link below can see the empty form and submit a response.": "Cualquiera que acceda al siguiente enlace puede ver el formulario vacío y enviar una respuesta.",
"Code copied to clipboard": "Código copiado al portapapeles",
"Link copied to clipboard": "Enlace copiado al portapapeles",
"Preview": "Vista previa",
"Reset form": "Restablecer el formulario",
"Save your document to publish this form.": "Guarde el documento para publicar este formulario.",
"Share this form": "Compartir este formulario",
"View": "Ver"
},
"WelcomeCoachingCall": {
"free coaching call": "llamada gratuita de asesoramiento",
@ -1442,5 +1463,17 @@
"This week": "Esta semana",
"This year": "Este año",
"Today": "Hoy"
},
"MappedFieldsConfig": {
"Clear": "Limpiar",
"Mapped": "Mapeado",
"Select All": "Seleccionar todo",
"Map fields": "Campos del mapa",
"Unmap fields": "Anular asignación de campos",
"Unmapped": "Sin mapear"
},
"Section": {
"Insert section above": "Insertar la sección anterior",
"Insert section below": "Insertar la sección siguiente"
}
}

42
static/locales/ru.client.json
View File

@ -293,7 +293,12 @@
"Save Document": "Сохранить документ",
"Work on a Copy": "Работа над копией",
"Share": "Поделиться",
"Download...": "Скачать..."
"Download...": "Скачать...",
"Comma Separated Values (.csv)": "Значения, разделенные запятыми (.csv)",
"Microsoft Excel (.xlsx)": "Microsoft Excel (.xlsx)",
"Tab Separated Values (.tsv)": "Значения, разделенные табуляцией (.tsv)",
"DOO Separated Values (.dsv)": "DOO Разделенные значения (.dsv)",
"Export as...": "Экспортировать как..."
},
"SortConfig": {
"Search Columns": "Поиск по столбцам",
@ -635,7 +640,8 @@
"Insert row": "Вставить строку",
"Insert row above": "Вставить строку выше",
"Delete": "Удалить",
"View as card": "Посмотреть как карточку"
"View as card": "Посмотреть как карточку",
"Use as table headers": "Использовать в качестве заголовков таблицы"
},
"RecordLayout": {
"Updating record layout.": "Обновление макета записи."
@ -716,7 +722,9 @@
"Submission": "Отправка",
"Submit another response": "Отправить другой ответ",
"Submit button label": "Название кнопки отправки",
"Success text": "Текст успешной отправки"
"Success text": "Текст успешной отправки",
"No field selected": "Нет выбранных полей",
"Select a field in the form widget to configure.": "Выберите поле в виджете формы для настройки."
},
"PermissionsWidget": {
"Allow All": "Разрешить все",
@ -1338,7 +1346,21 @@
"Publish": "Публиковать",
"Publish your form?": "Опубликовать форму?",
"Unpublish": "Отменить публикацию",
"Unpublish your form?": "Отменить публикацию формы?"
"Unpublish your form?": "Отменить публикацию формы?",
"Code copied to clipboard": "Код скопирован в буфер обмена",
"Copy code": "Скопировать код",
"Embed this form": "Встроить эту форму",
"Link copied to clipboard": "Ссылка скопирована в буфер обмена",
"Preview": "Предпросмотр",
"Save your document to publish this form.": "Сохраните документ, для публикации этой формы.",
"Share": "Поделиться",
"Share this form": "Поделиться этой формой",
"View": "Просмотр",
"Anyone with the link below can see the empty form and submit a response.": "Любой, у кого есть ссылка ниже, может увидеть пустую форму и отправить ответ.",
"Are you sure you want to reset your form?": "Вы уверены, что хотите сбросить форму?",
"Copy link": "Копировать ссылку",
"Reset": "Сброс",
"Reset form": "Сброс формы"
},
"WelcomeCoachingCall": {
"free coaching call": "бесплатный коуч-звонок",
@ -1387,5 +1409,17 @@
"This month": "Этот месяц",
"This week": "Эта неделя",
"This year": "Текущий год"
},
"MappedFieldsConfig": {
"Clear": "Очистить",
"Map fields": "Сопоставить поля",
"Mapped": "Сопоставлено",
"Select All": "Выбрать все",
"Unmap fields": "Отменить сопоставление полей",
"Unmapped": "Несопоставленные"
},
"Section": {
"Insert section above": "Вставить секцию выше",
"Insert section below": "Вставить секцию ниже"
}
}

42
static/locales/sl.client.json
View File

@ -282,7 +282,8 @@
"Duplicate rows_one": "Podvoji vrstico",
"Duplicate rows_other": "Podvoji vrstice",
"Insert row above": "Vstavi vrstico zgoraj",
"View as card": "Kartični pogled"
"View as card": "Kartični pogled",
"Use as table headers": "Uporabi kot glave tabel"
},
"Tools": {
"Delete": "Izbriši",
@ -612,7 +613,9 @@
"Enter text": "Vnesi besedilo",
"Redirect automatically after submission": "Po oddaji samodejno preusmeri",
"Enter redirect URL": "Vnesi preusmeritveni URL",
"Submit button label": "Oznaka gumba za pošiljanje"
"Submit button label": "Oznaka gumba za pošiljanje",
"Select a field in the form widget to configure.": "Izberi polje v gradniku obrazca, ki ga želiš konfigurirati.",
"No field selected": "Nobeno polje ni izbrano"
},
"FloatingPopup": {
"Maximize": "Povečajte",
@ -897,7 +900,12 @@
"Unsaved": "Neshranjeno",
"Save Document": "Shrani dokument",
"Save Copy": "Shrani kopijo",
"Return to {{termToUse}}": "Vrnitev na {{termToUse}}"
"Return to {{termToUse}}": "Vrnitev na {{termToUse}}",
"Export as...": "Izvozi kot ...",
"Comma Separated Values (.csv)": "Vrednosti, ločene z vejicami (.csv)",
"DOO Separated Values (.dsv)": "DOO ločene vrednosti (.dsv)",
"Microsoft Excel (.xlsx)": "Microsoft Excel (.xlsx)",
"Tab Separated Values (.tsv)": "Vrednosti, ločene s tabulatorji (.tsv)"
},
"NotifyUI": {
"Go to your free personal site": "Pojdite na brezplačno osebno spletno mesto",
@ -1317,7 +1325,21 @@
"Publish": "Objavi",
"Unpublish your form?": "Želiš preklicati objavo obrazca?",
"Publish your form?": "Želiš objaviti obrazec?",
"Unpublish": "Prekliči objavo"
"Unpublish": "Prekliči objavo",
"Link copied to clipboard": "Povezava je bila kopirana v odložišče",
"Preview": "Predogled",
"Reset": "Ponastaviti",
"Save your document to publish this form.": "Shrani dokument, da objaviš ta obrazec.",
"Share this form": "Deli ta dokument",
"View": "Pogled",
"Anyone with the link below can see the empty form and submit a response.": "Vsakdo s spodnjo povezavo si lahko ogleda prazen obrazec in odda odgovor.",
"Are you sure you want to reset your form?": "Ali si prepričan, da želiš ponastaviti obrazec?",
"Code copied to clipboard": "Koda je bila kopirana v odložišče",
"Copy code": "Kopiraj kodo",
"Copy link": "Kopiraj povezavo",
"Embed this form": "Vdelaj ta obrazec",
"Reset form": "Ponastavi obrazec",
"Share": "Deli"
},
"Menu": {
"Building blocks": "Gradniki",
@ -1387,5 +1409,17 @@
"This week": "Ta teden",
"This year": "To leto",
"Today": "Danes"
},
"MappedFieldsConfig": {
"Clear": "Počisti",
"Unmapped": "Nepreslikan",
"Map fields": "Preslikaj polja",
"Mapped": "Preslikan",
"Select All": "Izberi vse",
"Unmap fields": "Nepreslikana polja"
},
"Section": {
"Insert section above": "Vstavi razdelek zgoraj",
"Insert section below": "Vstavite razdelek spodaj"
}
}

118
test/server/Comm.ts
View File

@ -4,11 +4,11 @@ import {assert} from 'chai';
import * as http from 'http';
import {AddressInfo} from 'net';
import * as sinon from 'sinon';
import WebSocket from 'ws';
import * as path from 'path';
import * as tmp from 'tmp';
import {GristWSConnection, GristWSSettings} from 'app/client/components/GristWSConnection';
import {GristClientSocket, GristClientSocketOptions} from 'app/client/components/GristClientSocket';
import {Comm as ClientComm} from 'app/client/components/Comm';
import * as log from 'app/client/lib/log';
import {Comm} from 'app/server/lib/Comm';
@ -21,10 +21,28 @@ import {Sessions} from 'app/server/lib/Sessions';
import {TcpForwarder} from 'test/server/tcpForwarder';
import * as testUtils from 'test/server/testUtils';
import * as session from '@gristlabs/express-session';
import { Hosts, RequestOrgInfo } from 'app/server/lib/extractOrg';
const SQLiteStore = require('@gristlabs/connect-sqlite3')(session);
promisifyAll(SQLiteStore.prototype);
// Just enough implementation of Hosts to be able to fake using a custom host.
class FakeHosts {
public isCustomHost = false;
public get asHosts() { return this as unknown as Hosts; }
public async addOrgInfo<T extends http.IncomingMessage>(req: T): Promise<T & RequestOrgInfo> {
return Object.assign(req, {
isCustomHost: this.isCustomHost,
org: "example",
url: req.url!
});
}
}
describe('Comm', function() {
testUtils.setTmpLogLevel(process.env.VERBOSE ? 'debug' : 'warn');
@ -34,6 +52,7 @@ describe('Comm', function() {
let server: http.Server;
let sessions: Sessions;
let fakeHosts: FakeHosts;
let comm: Comm|null = null;
const sandbox = sinon.createSandbox();
@ -51,14 +70,19 @@ describe('Comm', function() {
function startComm(methods: {[name: string]: ClientMethod}) {
server = http.createServer();
comm = new Comm(server, {sessions});
fakeHosts = new FakeHosts();
comm = new Comm(server, {sessions, hosts: fakeHosts.asHosts});
comm.registerMethods(methods);
return listenPromise(server.listen(0, 'localhost'));
}
async function stopComm() {
comm?.destroyAllClients();
return fromCallback(cb => server.close(cb));
await comm?.testServerShutdown();
await fromCallback(cb => {
server.close(cb);
server.closeAllConnections();
});
}
const assortedMethods: {[name: string]: ClientMethod} = {
@ -95,34 +119,43 @@ describe('Comm', function() {
sandbox.restore();
});
function getMessages(ws: WebSocket, count: number): Promise<any[]> {
function getMessages(ws: GristClientSocket, count: number): Promise<any[]> {
return new Promise((resolve, reject) => {
const messages: object[] = [];
ws.on('error', reject);
ws.on('message', (msg: string) => {
messages.push(JSON.parse(msg));
ws.onerror = (err) => {
ws.onmessage = null;
reject(err);
};
ws.onmessage = (data: string) => {
messages.push(JSON.parse(data));
if (messages.length >= count) {
ws.onerror = null;
ws.onmessage = null;
resolve(messages);
ws.removeListener('error', reject);
ws.removeAllListeners('message');
}
});
};
});
}
/**
* Returns a promise for the connected websocket.
*/
function connect() {
const ws = new WebSocket('ws://localhost:' + (server.address() as AddressInfo).port);
return new Promise<WebSocket>((resolve, reject) => {
ws.on('open', () => resolve(ws));
ws.on('error', reject);
function connect(options?: GristClientSocketOptions): Promise<GristClientSocket> {
const ws = new GristClientSocket('ws://localhost:' + (server.address() as AddressInfo).port, options);
return new Promise<GristClientSocket>((resolve, reject) => {
ws.onopen = () => {
ws.onerror = null;
resolve(ws);
};
ws.onerror = (err) => {
ws.onopen = null;
reject(err);
};
});
}
describe("server methods", function() {
let ws: WebSocket;
let ws: GristClientSocket;
beforeEach(async function() {
await startComm(assortedMethods);
ws = await connect();
@ -370,7 +403,8 @@ describe('Comm', function() {
// Intercept the call to _onClose to know when it occurs, since we are trying to hit a
// situation where 'close' and 'failedSend' events happen in either order.
const stubOnClose = sandbox.stub(Client.prototype as any, '_onClose')
.callsFake(function(this: Client) {
.callsFake(async function(this: Client) {
if (!options.closeHappensFirst) { await delay(10); }
eventsSeen.push('close');
return (stubOnClose as any).wrappedMethod.apply(this, arguments);
});
@ -462,6 +496,9 @@ describe('Comm', function() {
if (options.useSmallMsgs) {
assert.deepEqual(eventsSeen, ['close']);
} else {
// Make sure to have waited long enough for the 'close' event we may have delayed
await delay(20);
// Large messages now cause a send to fail, after filling up buffer, and close the socket.
assert.deepEqual(eventsSeen, ['close', 'close']);
}
@ -490,17 +527,54 @@ describe('Comm', function() {
assert.deepEqual(eventSpy.getCalls().map(call => call.args[0].n), [n - 1]);
}
});
describe("Allowed Origin", function() {
beforeEach(async function () {
await startComm(assortedMethods);
});
afterEach(async function() {
await stopComm();
});
async function checkOrigin(headers: { origin: string, host: string }, allowed: boolean) {
const promise = connect({ headers });
if (allowed) {
await assert.isFulfilled(promise, `${headers.host} should allow ${headers.origin}`);
} else {
await assert.isRejected(promise, /.*/, `${headers.host} should reject ${headers.origin}`);
}
}
it('origin should match base domain of host', async () => {
await checkOrigin({origin: "https://www.toto.com", host: "worker.example.com"}, false);
await checkOrigin({origin: "https://badexample.com", host: "worker.example.com"}, false);
await checkOrigin({origin: "https://bad.com/example.com", host: "worker.example.com"}, false);
await checkOrigin({origin: "https://front.example.com", host: "worker.example.com"}, true);
await checkOrigin({origin: "https://front.example.com:3000", host: "worker.example.com"}, true);
await checkOrigin({origin: "https://example.com", host: "example.com"}, true);
});
it('with custom domains, origin should match the full hostname', async () => {
fakeHosts.isCustomHost = true;
// For a request to a custom domain, the full hostname must match.
await checkOrigin({origin: "https://front.example.com", host: "worker.example.com"}, false);
await checkOrigin({origin: "https://front.example.com", host: "front.example.com"}, true);
await checkOrigin({origin: "https://front.example.com:3000", host: "front.example.com"}, true);
});
});
});
// Waits for condFunc() to return true, for up to timeoutMs milliseconds, sleeping for stepMs
// between checks. Returns true if succeeded, false if failed.
async function waitForCondition(condFunc: () => boolean, timeoutMs = 1000, stepMs = 10): Promise<boolean> {
// between checks. Returns if succeeded, throws if failed.
async function waitForCondition(condFunc: () => boolean, timeoutMs = 1000, stepMs = 10): Promise<void> {
const end = Date.now() + timeoutMs;
while (Date.now() < end) {
if (condFunc()) { return true; }
if (condFunc()) { return; }
await delay(stepMs);
}
return false;
throw new Error(`Condition not met after ${timeoutMs}ms: ${condFunc.toString()}`);
}
// Returns a range of count consecutive numbers starting with start.
@ -513,7 +587,7 @@ function getWSSettings(docWorkerUrl: string): GristWSSettings {
let clientId: string = 'clientid-abc';
let counter: number = 0;
return {
makeWebSocket(url: string): any { return new WebSocket(url, undefined, {}); },
makeWebSocket(url: string): any { return new GristClientSocket(url); },
async getTimezone() { return 'UTC'; },
getPageUrl() { return "http://localhost"; },
async getDocWorkerUrl() { return docWorkerUrl; },

21
test/server/gristClient.ts
View File

@ -4,7 +4,7 @@ import { SchemaTypes } from 'app/common/schema';
import { FlexServer } from 'app/server/lib/FlexServer';
import axios from 'axios';
import pick = require('lodash/pick');
import WebSocket from 'ws';
import {GristClientSocket} from 'app/client/components/GristClientSocket';
interface GristRequest {
reqId: number;
@ -34,9 +34,9 @@ export class GristClient {
private _consumer: () => void;
private _ignoreTrivialActions: boolean = false;
constructor(public ws: any) {
ws.onmessage = (data: any) => {
const msg = pick(JSON.parse(data.data),
constructor(public ws: GristClientSocket) {
ws.onmessage = (data: string) => {
const msg = pick(JSON.parse(data),
['reqId', 'error', 'errorCode', 'data', 'type', 'docFD']);
if (this._ignoreTrivialActions && msg.type === 'docUserAction' &&
msg.data?.actionGroup?.internal === true &&
@ -149,7 +149,6 @@ export class GristClient {
}
public async close() {
this.ws.terminate();
this.ws.close();
}
@ -180,17 +179,19 @@ export async function openClient(server: FlexServer, email: string, org: string,
} else {
headers[emailHeader] = email;
}
const ws = new WebSocket('ws://localhost:' + server.getOwnPort() + `/o/${org}`, {
const ws = new GristClientSocket('ws://localhost:' + server.getOwnPort() + `/o/${org}`, {
headers
});
const client = new GristClient(ws);
await new Promise(function(resolve, reject) {
ws.on('open', function() {
ws.onopen = function() {
ws.onerror = null;
resolve(ws);
});
ws.on('error', function(err: any) {
};
ws.onerror = function(err: Error) {
ws.onopen = null;
reject(err);
});
};
});
return client;
}

14
test/server/lib/DocApi.ts
View File

@ -4879,11 +4879,17 @@ function testDocApi() {
delete chimpyConfig.headers["X-Requested-With"];
delete anonConfig.headers["X-Requested-With"];
// Target a more realistic Host than "localhost:port"
anonConfig.headers.Host = chimpyConfig.headers.Host = 'api.example.com';
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
const data = {records: [{fields: {}}]};
const data = { records: [{ fields: {} }] };
const allowedOrigin = 'http://front.example.com';
const forbiddenOrigin = 'http://evil.com';
// Normal same origin requests
anonConfig.headers.Origin = serverUrl;
anonConfig.headers.Origin = allowedOrigin;
let response: AxiosResponse;
for (response of [
await axios.post(url, data, anonConfig),
@ -4893,13 +4899,13 @@ function testDocApi() {
assert.equal(response.status, 200);
assert.equal(response.headers['access-control-allow-methods'], 'GET, PATCH, PUT, POST, DELETE, OPTIONS');
assert.equal(response.headers['access-control-allow-headers'], 'Authorization, Content-Type, X-Requested-With');
assert.equal(response.headers['access-control-allow-origin'], serverUrl);
assert.equal(response.headers['access-control-allow-origin'], allowedOrigin);
assert.equal(response.headers['access-control-allow-credentials'], 'true');
}
// Cross origin requests from untrusted origin.
for (const config of [anonConfig, chimpyConfig]) {
config.headers.Origin = "https://evil.com/";
config.headers.Origin = forbiddenOrigin;
for (response of [
await axios.post(url, data, config),
await axios.get(url, config),

170
test/server/lib/GristSockets.ts Normal file
View File

@ -0,0 +1,170 @@
import { assert } from 'chai';
import * as http from 'http';
import { GristClientSocket } from 'app/client/components/GristClientSocket';
import { GristSocketServer } from 'app/server/lib/GristSocketServer';
import { fromCallback, listenPromise } from 'app/server/lib/serverUtils';
import { AddressInfo } from 'net';
import httpProxy from 'http-proxy';
describe(`GristSockets`, function () {
for (const webSocketsSupported of [true, false]) {
describe(`when the networks ${webSocketsSupported ? "supports" : "does not support"} WebSockets`, function () {
let server: http.Server | null;
let serverPort: number;
let socketServer: GristSocketServer | null;
let proxy: httpProxy | null;
let proxyServer: http.Server | null;
let proxyPort: number;
let wsAddress: string;
beforeEach(async function () {
await startSocketServer();
await startProxyServer();
});
afterEach(async function () {
await stopProxyServer();
await stopSocketServer();
});
async function startSocketServer() {
server = http.createServer((req, res) => res.writeHead(404).end());
socketServer = new GristSocketServer(server);
await listenPromise(server.listen(0, 'localhost'));
serverPort = (server.address() as AddressInfo).port;
}
async function stopSocketServer() {
await fromCallback(cb => socketServer?.close(cb));
await fromCallback(cb => { server?.close(); server?.closeAllConnections(); server?.on("close", cb); });
socketServer = server = null;
}
// Start an HTTP proxy that supports WebSockets or not
async function startProxyServer() {
proxy = httpProxy.createProxy({
target: `http://localhost:${serverPort}`,
ws: webSocketsSupported,
timeout: 1000,
});
proxy.on('error', () => { });
proxyServer = http.createServer();
if (webSocketsSupported) {
// prevent non-WebSocket requests
proxyServer.on('request', (req, res) => res.writeHead(404).end());
// proxy WebSocket requests
proxyServer.on('upgrade', (req, socket, head) => proxy!.ws(req, socket, head));
} else {
// proxy non-WebSocket requests
proxyServer.on('request', (req, res) => proxy!.web(req, res));
// don't leave WebSocket connection attempts hanging
proxyServer.on('upgrade', (req, socket, head) => socket.destroy());
}
await listenPromise(proxyServer.listen(0, 'localhost'));
proxyPort = (proxyServer.address() as AddressInfo).port;
wsAddress = `ws://localhost:${proxyPort}`;
}
async function stopProxyServer() {
if (proxy) {
proxy.close();
proxy = null;
}
if (proxyServer) {
const server = proxyServer;
await fromCallback(cb => { server.close(cb); server.closeAllConnections(); });
}
proxyServer = null;
}
function getMessages(ws: GristClientSocket, count: number): Promise<string[]> {
return new Promise((resolve, reject) => {
const messages: string[] = [];
ws.onerror = (err) => {
ws.onerror = ws.onmessage = null;
reject(err);
};
ws.onmessage = (data: string) => {
messages.push(data);
if (messages.length >= count) {
ws.onerror = ws.onmessage = null;
resolve(messages);
}
};
});
}
/**
* Returns a promise for the connected websocket.
*/
function connectClient(url: string): Promise<GristClientSocket> {
const socket = new GristClientSocket(url);
return new Promise<GristClientSocket>((resolve, reject) => {
socket.onopen = () => {
socket.onerror = null;
resolve(socket);
};
socket.onerror = (err) => {
socket.onopen = null;
reject(err);
};
});
}
it("should expose initial request", async function () {
const connectionPromise = new Promise<http.IncomingMessage>((resolve) => {
socketServer!.onconnection = (socket, req) => {
resolve(req);
};
});
const clientWs = new GristClientSocket(wsAddress + "/path?query=value", {
headers: { "cookie": "session=1234" }
});
const req = await connectionPromise;
clientWs.close();
// Engine.IO may append extra query parameters, so we check only the start of the URL
assert.match(req.url!, /^\/path\?query=value/);
assert.equal(req.headers.cookie, "session=1234");
});
it("should receive and send messages", async function () {
socketServer!.onconnection = (socket, req) => {
socket.onmessage = (data) => {
socket.send("hello, " + data);
};
};
const clientWs = await connectClient(wsAddress);
clientWs.send("world");
assert.deepEqual(await getMessages(clientWs, 1), ["hello, world"]);
clientWs.close();
});
it("should invoke send callbacks", async function () {
const connectionPromise = new Promise<void>((resolve) => {
socketServer!.onconnection = (socket, req) => {
socket.send("hello", () => resolve());
};
});
const clientWs = await connectClient(wsAddress);
await connectionPromise;
clientWs.close();
});
it("should emit close event for client", async function () {
const clientWs = await connectClient(wsAddress);
const closePromise = new Promise<void>(resolve => {
clientWs.onclose = resolve;
});
clientWs.close();
await closePromise;
});
});
}
});

4
test/server/lib/ManyFetches.ts
View File

@ -12,7 +12,7 @@ import {createTestDir, EnvironmentSnapshot, setTmpLogLevel} from 'test/server/te
import {assert} from 'chai';
import * as cookie from 'cookie';
import fetch from 'node-fetch';
import WebSocket from 'ws';
import {GristClientSocket} from 'app/client/components/GristClientSocket';
describe('ManyFetches', function() {
this.timeout(30000);
@ -244,7 +244,7 @@ describe('ManyFetches', function() {
return function createConnectionFunc() {
let clientId: string = '0';
return GristWSConnection.create(null, {
makeWebSocket(url: string): any { return new WebSocket(url, undefined, { headers }); },
makeWebSocket(url: string) { return new GristClientSocket(url, { headers }); },
getTimezone() { return Promise.resolve('UTC'); },
getPageUrl() { return pageUrl; },
getDocWorkerUrl() { return Promise.resolve(docWorkerUrl); },

2
test/test_under_docker.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# This runs browser tests with the server started using docker, to
# catch any configuration problems.

105
yarn.lock
View File

@ -544,6 +544,11 @@
resolved "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz"
integrity sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==
"@socket.io/component-emitter@~3.1.0":
version "3.1.0"
resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553"
integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg==
"@sqltools/formatter@^1.2.2":
version "1.2.3"
resolved "https://registry.npmjs.org/@sqltools/formatter/-/formatter-1.2.3.tgz"
@ -626,6 +631,18 @@
resolved "https://registry.npmjs.org/@types/content-disposition/-/content-disposition-0.5.2.tgz"
integrity sha1-GMyw/RJoCOSYqCDLgUnAByorkGY=
"@types/cookie@^0.4.1":
version "0.4.1"
resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d"
integrity sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==
"@types/cors@^2.8.12":
version "2.8.17"
resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.17.tgz#5d718a5e494a8166f569d986794e49c48b216b2b"
integrity sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==
dependencies:
"@types/node" "*"
"@types/d3@^3":
version "3.5.44"
resolved "https://registry.npmjs.org/@types/d3/-/d3-3.5.44.tgz"
@ -844,6 +861,13 @@
resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.9.tgz#02d013de7058cea16d36168ef2fc653464cfbad4"
integrity sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==
"@types/node@>=10.0.0":
version "20.11.16"
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.16.tgz#4411f79411514eb8e2926f036c86c9f0e4ec6708"
integrity sha512-gKb0enTmRCzXSSUJDq6/sPcqrfCv2mkkG6Jt/clpn5eiCbKTY+SgZUxo+p8ZKMof5dCp9vHQUAB7wOUTod22wQ==
dependencies:
undici-types "~5.26.4"
"@types/node@^14.0.1":
version "14.17.6"
resolved "https://registry.npmjs.org/@types/node/-/node-14.17.6.tgz"
@ -988,10 +1012,10 @@
dependencies:
"@types/node" "*"
"@types/ws@^6":
version "6.0.4"
resolved "https://registry.npmjs.org/@types/ws/-/ws-6.0.4.tgz"
integrity sha512-PpPrX7SZW9re6+Ha8ojZG4Se8AZXgf0GK6zmfqEuCsY49LFDNXO3SByp44X3dFEqtB73lkCDAdUazhAjVPiNwg==
"@types/ws@^8":
version "8.5.10"
resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.10.tgz#4acfb517970853fa6574a3a6886791d04a396787"
integrity sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==
dependencies:
"@types/node" "*"
@ -1253,7 +1277,7 @@ accept-language-parser@1.5.0:
resolved "https://registry.npmjs.org/accept-language-parser/-/accept-language-parser-1.5.0.tgz"
integrity sha1-iHfFQECo3LWeCgfZwf3kIpgzR5E=
accepts@~1.3.8:
accepts@~1.3.4, accepts@~1.3.8:
version "1.3.8"
resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e"
integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==
@ -1667,6 +1691,11 @@ base64-js@^1.2.0, base64-js@^1.3.0, base64-js@^1.3.1:
resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz"
integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
base64id@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6"
integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==
basic-auth@~2.0.0:
version "2.0.1"
resolved "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz"
@ -2573,11 +2602,24 @@ cookie@0.5.0:
resolved "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz"
integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==
cookie@~0.4.1:
version "0.4.2"
resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432"
integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==
core-util-is@~1.0.0:
version "1.0.3"
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85"
integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==
cors@~2.8.5:
version "2.8.5"
resolved "https://registry.yarnpkg.com/cors/-/cors-2.8.5.tgz#eac11da51592dd86b9f06f6e7ac293b3df875d29"
integrity sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==
dependencies:
object-assign "^4"
vary "^1"
crc-32@^1.2.0:
version "1.2.0"
resolved "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz"
@ -2737,7 +2779,7 @@ debug@4:
dependencies:
ms "2.1.2"
debug@4.3.4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4:
debug@4.3.4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1:
version "4.3.4"
resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz"
integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
@ -3072,6 +3114,38 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1:
dependencies:
once "^1.4.0"
engine.io-client@^6.5.3:
version "6.5.3"
resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.5.3.tgz#4cf6fa24845029b238f83c628916d9149c399bc5"
integrity sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==
dependencies:
"@socket.io/component-emitter" "~3.1.0"
debug "~4.3.1"
engine.io-parser "~5.2.1"
ws "~8.11.0"
xmlhttprequest-ssl "~2.0.0"
engine.io-parser@~5.2.1:
version "5.2.2"
resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.2.2.tgz#37b48e2d23116919a3453738c5720455e64e1c49"
integrity sha512-RcyUFKA93/CXH20l4SoVvzZfrSDMOTUS3bWVpTt2FuFP+XYrL8i8oonHP7WInRyVHXh0n/ORtoeiE1os+8qkSw==
engine.io@^6.5.4:
version "6.5.4"
resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.5.4.tgz#6822debf324e781add2254e912f8568508850cdc"
integrity sha512-KdVSDKhVKyOi+r5uEabrDLZw2qXStVvCsEB/LN3mw4WFi6Gx50jTyuxYVCwAAC0U46FdnzP/ScKRBTXb/NiEOg==
dependencies:
"@types/cookie" "^0.4.1"
"@types/cors" "^2.8.12"
"@types/node" ">=10.0.0"
accepts "~1.3.4"
base64id "2.0.0"
cookie "~0.4.1"
cors "~2.8.5"
debug "~4.3.1"
engine.io-parser "~5.2.1"
ws "~8.11.0"
enhanced-resolve@^5.9.3:
version "5.9.3"
resolved "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.9.3.tgz"
@ -5894,7 +5968,7 @@ nwsapi@^2.2.7:
resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.7.tgz#738e0707d3128cb750dddcfe90e4610482df0f30"
integrity sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==
object-assign@^4.0.1, object-assign@^4.1.1:
object-assign@^4, object-assign@^4.0.1, object-assign@^4.1.1:
version "4.1.1"
resolved "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz"
integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=
@ -7863,6 +7937,11 @@ underscore@1.12.1, underscore@>=1.8.3, underscore@^1.8.0:
resolved "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz"
integrity sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==
undici-types@~5.26.4:
version "5.26.5"
resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
unique-filename@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230"
@ -8053,7 +8132,7 @@ value-or-function@^3.0.0:
resolved "https://registry.yarnpkg.com/value-or-function/-/value-or-function-3.0.0.tgz#1c243a50b595c1be54a754bfece8563b9ff8d813"
integrity sha512-jdBB2FrWvQC/pnPtIqcLsMaQgjhdb6B7tk1MMyTKapox+tQZbdRP4uLxu/JY0t7fbfDCUMnuelzEYv5GsxHhdg==
vary@~1.1.2:
vary@^1, vary@~1.1.2:
version "1.1.2"
resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz"
integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=
@ -8369,6 +8448,11 @@ ws@^8.14.2:
resolved "https://registry.yarnpkg.com/ws/-/ws-8.15.0.tgz#db080a279260c5f532fc668d461b8346efdfcf86"
integrity sha512-H/Z3H55mrcrgjFwI+5jKavgXvwQLtfPCUEp6pi35VhoB0pfcHnSoyuTzkBEZpzq49g1193CUEwIvmsjcotenYw==
ws@~8.11.0:
version "8.11.0"
resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143"
integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==
xdg-basedir@^4.0.0:
version "4.0.0"
resolved "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz"
@ -8438,6 +8522,11 @@ xmldom@^0.1.0, xmldom@~0.1.15:
resolved "https://registry.npmjs.org/xmldom/-/xmldom-0.1.31.tgz"
integrity sha512-yS2uJflVQs6n+CyjHoaBmVSqIDevTAWrzMmjG1Gc7h1qQ7uVozNhEPJAwZXWyGQ/Gafo3fCwrcaokezLPupVyQ==
xmlhttprequest-ssl@~2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67"
integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A==
xpath.js@>=0.0.3:
version "1.1.0"
resolved "https://registry.npmjs.org/xpath.js/-/xpath.js-1.1.0.tgz"