Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) updates from grist-core

Browse Source
This commit is contained in:
Paul Fitzpatrick
2024-05-23 13:27:59 -04:00
parent 31722de2eb 53cfc6148d
commit 76a43129f1
42 changed files with 2647 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/client/models/entities/ViewSectionRec.ts
View File

@@ -460,7 +460,9 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel):
// - Widget type description (if not grid)
// All concatenated separated by space.
this.defaultWidgetTitle = this.autoDispose(ko.pureComputed(() => {
const widgetTypeDesc = this.parentKey() !== 'record' ? `${getWidgetTypes(this.parentKey.peek() as any).label}` : '';
const widgetTypeDesc = this.parentKey() !== 'record'
? `${getWidgetTypes(this.parentKey.peek() as any).getLabel()}`
: '';
const table = this.table();
return [
table.tableNameDef()?.toUpperCase(), // Due to ACL this can be null.

42
app/client/ui/AdminPanel.ts
View File

@@ -105,6 +105,13 @@ export class AdminPanel extends Disposable {
value: this._buildSandboxingDisplay(owner),
expandedContent: this._buildSandboxingNotice(),
}),
dom.create(AdminSectionItem, {
id: 'authentication',
name: t('Authentication'),
description: t('Current authentication method'),
value: this._buildAuthenticationDisplay(owner),
expandedContent: this._buildAuthenticationNotice(owner),
})
]),
dom.create(AdminSection, t('Version'), [
@@ -156,6 +163,37 @@ isolated from other documents and isolated from the network.'),
];
}
private _buildAuthenticationDisplay(owner: IDisposableOwner) {
return dom.domComputed(
use => {
const req = this._checks.requestCheckById(use, 'authentication');
const result = req ? use(req.result) : undefined;
if (!result) {
return cssValueLabel(cssErrorText('unavailable'));
}
const { success, details } = result;
const loginSystemId = details?.loginSystemId;
if (!success || !loginSystemId) {
return cssValueLabel(cssErrorText('auth error'));
}
if (loginSystemId === 'no-logins') {
return cssValueLabel(cssDangerText('no authentication'));
}
return cssValueLabel(cssHappyText(loginSystemId));
}
);
}
private _buildAuthenticationNotice(owner: IDisposableOwner) {
return t('Grist allows different types of authentication to be configured, including SAML and OIDC. \
We recommend enabling one of these if Grist is accessible over the network or being made available \
to multiple people.');
}
private _buildUpdates(owner: MultiHolder) {
// We can be in those states:
enum State {
@@ -446,6 +484,10 @@ const cssErrorText = styled('span', `
color: ${theme.errorText};
`);
export const cssDangerText = styled('div', `
color: ${theme.dangerText};
`);
const cssHappyText = styled('span', `
color: ${theme.controlFg};
`);

1
app/client/ui/AdminPanelCss.ts
View File

@@ -124,6 +124,7 @@ const cssItemName = styled('div', `
font-weight: bold;
display: flex;
align-items: center;
margin-right: 14px;
font-size: ${vars.largeFontSize};
padding-left: 24px;
&-prefixed {

5
app/client/ui/ColumnFilterMenu.ts
View File

@@ -130,7 +130,10 @@ export function columnFilterMenu(owner: IDisposableOwner, opts: IFilterMenuOptio
dom.onDispose(() => cancel ? doCancel() : doSave()),
dom.onKeyDown({
Enter: () => onClose(),
Escape: () => onClose(),
Escape: () => {
cancel = true;
onClose();
},
}),
// Filter by range

8
app/client/ui/HomeLeftPane.ts
View File

@@ -146,6 +146,14 @@ export function createHomeLeftPane(leftPanelOpen: Observable<boolean>, home: Hom
) : null
),
createHelpTools(home.app),
(commonUrls.termsOfService ?
cssPageEntry(
cssPageLink(cssPageIcon('Memo'), cssLinkText(t("Terms of service")),
{ href: commonUrls.termsOfService, target: '_blank' },
testId('dm-tos'),
),
) : null
),
)
)
);

8
app/client/ui/PageWidgetPicker.ts
View File

@@ -205,7 +205,7 @@ export function buildPageWidgetPicker(
// If savePromise throws an error, before or after timeout, we let the error propagate as it
// should be handle by the caller.
if (await isLongerThan(savePromise, DELAY_BEFORE_SPINNER_MS)) {
const label = getWidgetTypes(type).label;
const label = getWidgetTypes(type).getLabel();
await spinnerModal(t("Building {{- label}} widget", { label }), savePromise);
}
}
@@ -317,12 +317,12 @@ export class PageWidgetSelect extends Disposable {
cssPanel(
header(t("Select Widget")),
sectionTypes.map((value) => {
const {label, icon: iconName} = getWidgetTypes(value);
const widgetInfo = getWidgetTypes(value);
const disabled = computed(this._value.table, (use, tid) => this._isTypeDisabled(value, tid));
return cssEntry(
dom.autoDispose(disabled),
cssTypeIcon(iconName),
label,
cssTypeIcon(widgetInfo.icon),
widgetInfo.getLabel(),
dom.on('click', () => !disabled.get() && this._selectType(value)),
cssEntry.cls('-selected', (use) => use(this._value.type) === value),
cssEntry.cls('-disabled', disabled),

6
app/client/ui/RightPanel.ts
View File

@@ -38,7 +38,7 @@ import {PredefinedCustomSectionConfig} from "app/client/ui/PredefinedCustomSecti
import {cssLabel} from 'app/client/ui/RightPanelStyles';
import {linkId, NoLink, selectBy} from 'app/client/ui/selectBy';
import {VisibleFieldsConfig} from 'app/client/ui/VisibleFieldsConfig';
import {getTelemetryWidgetTypeFromVS, widgetTypesMap} from "app/client/ui/widgetTypesMap";
import {getTelemetryWidgetTypeFromVS, getWidgetTypes} from "app/client/ui/widgetTypesMap";
import {basicButton, primaryButton} from 'app/client/ui2018/buttons';
import {buttonSelect} from 'app/client/ui2018/buttonSelect';
import {labeledSquareCheckbox} from 'app/client/ui2018/checkbox';
@@ -220,10 +220,10 @@ export class RightPanel extends Disposable {
private _buildStandardHeader() {
return dom.maybe(this._pageWidgetType, (type) => {
const widgetInfo = widgetTypesMap.get(type) || {label: 'Table', icon: 'TypeTable'};
const widgetInfo = getWidgetTypes(type);
const fieldInfo = getFieldType(type);
return [
cssTopBarItem(cssTopBarIcon(widgetInfo.icon), widgetInfo.label,
cssTopBarItem(cssTopBarIcon(widgetInfo.icon), widgetInfo.getLabel(),
cssTopBarItem.cls('-selected', (use) => use(this._topTab) === 'pageWidget'),
dom.on('click', () => this._topTab.set("pageWidget")),
testId('right-tab-pagewidget')),

22
app/client/ui/widgetTypesMap.ts
View File

@@ -3,21 +3,25 @@ import {ViewSectionRec} from "app/client/models/entities/ViewSectionRec";
import {IPageWidget} from "app/client/ui/PageWidgetPicker";
import {IconName} from "app/client/ui2018/IconList";
import {IWidgetType} from "app/common/widgetTypes";
import {makeT} from 'app/client/lib/localization';
const t = makeT('widgetTypesMap');
export const widgetTypesMap = new Map<IWidgetType, IWidgetTypeInfo>([
['record', {label: 'Table', icon: 'TypeTable'}],
['single', {label: 'Card', icon: 'TypeCard'}],
['detail', {label: 'Card List', icon: 'TypeCardList'}],
['chart', {label: 'Chart', icon: 'TypeChart'}],
['form', {label: 'Form', icon: 'Board'}],
['custom', {label: 'Custom', icon: 'TypeCustom'}],
['custom.calendar', {label: 'Calendar', icon: 'TypeCalendar'}],
['record', {name: 'Table', icon: 'TypeTable', getLabel: () => t('Table')}],
['single', {name: 'Card', icon: 'TypeCard', getLabel: () => t('Card')}],
['detail', {name: 'Card List', icon: 'TypeCardList', getLabel: () => t('Card List')}],
['chart', {name: 'Chart', icon: 'TypeChart', getLabel: () => t('Chart')}],
['form', {name: 'Form', icon: 'Board', getLabel: () => t('Form')}],
['custom', {name: 'Custom', icon: 'TypeCustom', getLabel: () => t('Custom')}],
['custom.calendar', {name: 'Calendar', icon: 'TypeCalendar', getLabel: () => t('Calendar')}],
]);
// Widget type info.
export interface IWidgetTypeInfo {
label: string;
name: string;
icon: IconName;
getLabel: () => string;
}
// Returns the widget type info for sectionType, or the one for 'record' if sectionType is null.
@@ -46,7 +50,7 @@ export function getTelemetryWidgetTypeFromPageWidget(widget: IPageWidget) {
}
function getTelemetryWidgetType(type: IWidgetType, options: GetTelemetryWidgetTypeOptions = {}) {
let telemetryWidgetType: string | undefined = widgetTypesMap.get(type)?.label;
let telemetryWidgetType: string | undefined = widgetTypesMap.get(type)?.name;
if (!telemetryWidgetType) { return undefined; }
if (options.isNewTable) {

3
app/common/BootProbe.ts
View File

@@ -6,7 +6,8 @@ export type BootProbeIds =
'reachable' |
'host-header' |
'sandboxing' |
'system-user'
'system-user' |
'authentication'
;
export interface BootProbeResult {

49
app/common/UserAPI.ts
View File

@@ -780,11 +780,11 @@ export class UserAPIImpl extends BaseAPI implements UserAPI {
}
public async getWorker(key: string): Promise<string> {
const json = await this.requestJson(`${this._url}/api/worker/${key}`, {
const json = (await this.requestJson(`${this._url}/api/worker/${key}`, {
method: 'GET',
credentials: 'include'
});
return getDocWorkerUrl(this._homeUrl, json);
})) as PublicDocWorkerUrlInfo;
return getPublicDocWorkerUrl(this._homeUrl, json);
}
public async getWorkerAPI(key: string): Promise<DocWorkerAPI> {
@@ -1163,6 +1163,27 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
}
}
/**
* Represents information to build public doc worker url.
*
* Structure that may contain either **exclusively**:
* - a selfPrefix when no pool of doc worker exist.
* - a public doc worker url otherwise.
*/
export type PublicDocWorkerUrlInfo = {
selfPrefix: string;
docWorkerUrl: null;
} | {
selfPrefix: null;
docWorkerUrl: string;
}
export function getUrlFromPrefix(homeUrl: string, prefix: string) {
const url = new URL(homeUrl);
url.pathname = prefix + url.pathname;
return url.href;
}
/**
* Get a docWorkerUrl from information returned from backend. When the backend
* is fully configured, and there is a pool of workers, this is straightforward,
@@ -1171,19 +1192,13 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
* use the homeUrl of the backend, with extra path prefix information
* given by selfPrefix. At the time of writing, the selfPrefix contains a
* doc-worker id, and a tag for the codebase (used in consistency checks).
*
* @param {string} homeUrl
* @param {string} docWorkerInfo The information to build the public doc worker url
* (result of the call to /api/worker/:docId)
*/
export function getDocWorkerUrl(homeUrl: string, docWorkerInfo: {
docWorkerUrl: string|null,
selfPrefix?: string,
}): string {
if (!docWorkerInfo.docWorkerUrl) {
if (!docWorkerInfo.selfPrefix) {
// This should never happen.
throw new Error('missing selfPrefix for docWorkerUrl');
}
const url = new URL(homeUrl);
url.pathname = docWorkerInfo.selfPrefix + url.pathname;
return url.href;
}
return docWorkerInfo.docWorkerUrl;
export function getPublicDocWorkerUrl(homeUrl: string, docWorkerInfo: PublicDocWorkerUrlInfo) {
return docWorkerInfo.selfPrefix !== null ?
getUrlFromPrefix(homeUrl, docWorkerInfo.selfPrefix) :
docWorkerInfo.docWorkerUrl;
}

40
app/common/gristUrls.ts
View File

@@ -91,6 +91,7 @@ export const commonUrls = {
helpAPI: 'https://support.getgrist.com/api',
freeCoachingCall: getFreeCoachingCallUrl(),
contactSupport: getContactSupportUrl(),
termsOfService: getTermsOfServiceUrl(),
plans: "https://www.getgrist.com/pricing",
sproutsProgram: "https://www.getgrist.com/sprouts-program",
contact: "https://www.getgrist.com/contact",
@@ -187,10 +188,23 @@ export interface OrgUrlInfo {
orgInPath?: string; // If /o/{orgInPath} should be used to access the requested org.
}
function isDocInternalUrl(host: string) {
if (!process.env.APP_DOC_INTERNAL_URL) { return false; }
const internalUrl = new URL('/', process.env.APP_DOC_INTERNAL_URL);
return internalUrl.host === host;
function hostMatchesUrl(host?: string, url?: string) {
return host !== undefined && url !== undefined && new URL(url).host === host;
}
/**
* Returns true if:
* - the server is a home worker and the host matches APP_HOME_INTERNAL_URL;
* - or the server is a doc worker and the host matches APP_DOC_INTERNAL_URL;
*
* @param {string?} host The host to check
*/
function isOwnInternalUrlHost(host?: string) {
// Note: APP_HOME_INTERNAL_URL may also be defined in doc worker as well as in home worker
if (process.env.APP_HOME_INTERNAL_URL && hostMatchesUrl(host, process.env.APP_HOME_INTERNAL_URL)) {
return true;
}
return Boolean(process.env.APP_DOC_INTERNAL_URL) && hostMatchesUrl(host, process.env.APP_DOC_INTERNAL_URL);
}
/**
@@ -210,7 +224,11 @@ export function getHostType(host: string, options: {
const hostname = host.split(":")[0];
if (!options.baseDomain) { return 'native'; }
if (hostname === 'localhost' || isDocInternalUrl(host) || hostname.endsWith(options.baseDomain)) {
if (
hostname === 'localhost' ||
isOwnInternalUrlHost(host) ||
hostname.endsWith(options.baseDomain)
) {
return 'native';
}
return 'custom';
@@ -676,6 +694,9 @@ export interface GristLoadConfig {
// Url for support for the browser client to use.
helpCenterUrl?: string;
// Url for terms of service for the browser client to use
termsOfServiceUrl?: string;
// Url for free coaching call scheduling for the browser client to use.
freeCoachingCallUrl?: string;
@@ -887,6 +908,15 @@ export function getHelpCenterUrl(): string {
}
}
export function getTermsOfServiceUrl(): string|undefined {
if(isClient()) {
const gristConfig: GristLoadConfig = (window as any).gristConfig;
return gristConfig && gristConfig.termsOfServiceUrl || undefined;
} else {
return process.env.GRIST_TERMS_OF_SERVICE_URL || undefined;
}
}
export function getFreeCoachingCallUrl(): string {
const defaultUrl = "https://calendly.com/grist-team/grist-free-coaching-call";
if(isClient()) {

6
app/gen-server/lib/DocApiForwarder.ts
View File

@@ -104,7 +104,11 @@ export class DocApiForwarder {
url.pathname = removeTrailingSlash(docWorkerUrl.pathname) + url.pathname;
const headers: {[key: string]: string} = {
...getTransitiveHeaders(req),
// At this point, we have already checked and trusted the origin of the request.
// See FlexServer#addApiMiddleware(). So don't include the "Origin" header.
// Including this header also would break features like form submissions,
// as the "Host" header is not retrieved when calling getTransitiveHeaders().
...getTransitiveHeaders(req, { includeOrigin: false }),
'Content-Type': req.get('Content-Type') || 'application/json',
};
for (const key of ['X-Sort', 'X-Limit']) {

49
app/server/lib/AppEndpoint.ts
View File

@@ -9,17 +9,18 @@ import {ApiError} from 'app/common/ApiError';
import {getSlugIfNeeded, parseUrlId, SHARE_KEY_PREFIX} from 'app/common/gristUrls';
import {LocalPlugin} from "app/common/plugin";
import {TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME} from 'app/common/Telemetry';
import {Document as APIDocument} from 'app/common/UserAPI';
import {Document as APIDocument, PublicDocWorkerUrlInfo} from 'app/common/UserAPI';
import {Document} from "app/gen-server/entity/Document";
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {assertAccess, getTransitiveHeaders, getUserId, isAnonymousUser,
RequestWithLogin} from 'app/server/lib/Authorizer';
import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {customizeDocWorkerUrl, getWorker, useWorkerPool} from 'app/server/lib/DocWorkerUtils';
import {
customizeDocWorkerUrl, getDocWorkerInfoOrSelfPrefix, getWorker, useWorkerPool
} from 'app/server/lib/DocWorkerUtils';
import {expressWrap} from 'app/server/lib/expressWrap';
import {DocTemplate, GristServer} from 'app/server/lib/GristServer';
import {getCookieDomain} from 'app/server/lib/gristSessions';
import {getAssignmentId} from 'app/server/lib/idUtils';
import log from 'app/server/lib/log';
import {addOrgToPathIfNeeded, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils';
import {ISendAppPageOptions} from 'app/server/lib/sendAppPage';
@@ -48,32 +49,18 @@ export function attachAppEndpoint(options: AttachOptions): void {
app.get('/apiconsole', expressWrap(async (req, res) =>
sendAppPage(req, res, {path: 'apiconsole.html', status: 200, config: {}})));
app.get('/api/worker/:assignmentId([^/]+)/?*', expressWrap(async (req, res) => {
if (!useWorkerPool()) {
// Let the client know there is not a separate pool of workers,
// so they should continue to use the same base URL for accessing
// documents. For consistency, return a prefix to add into that
// URL, as there would be for a pool of workers. It would be nice
// to go ahead and provide the full URL, but that requires making
// more assumptions about how Grist is configured.
// Alternatives could be: have the client to send their base URL
// in the request; or use headers commonly added by reverse proxies.
const selfPrefix = "/dw/self/v/" + gristServer.getTag();
res.json({docWorkerUrl: null, selfPrefix});
return;
}
app.get('/api/worker/:docId([^/]+)/?*', expressWrap(async (req, res) => {
if (!trustOrigin(req, res)) { throw new Error('Unrecognized origin'); }
res.header("Access-Control-Allow-Credentials", "true");
if (!docWorkerMap) {
return res.status(500).json({error: 'no worker map'});
}
const assignmentId = getAssignmentId(docWorkerMap, req.params.assignmentId);
const {docStatus} = await getWorker(docWorkerMap, assignmentId, '/status');
if (!docStatus) {
return res.status(500).json({error: 'no worker'});
}
res.json({docWorkerUrl: customizeDocWorkerUrl(docStatus.docWorker.publicUrl, req)});
const {selfPrefix, docWorker} = await getDocWorkerInfoOrSelfPrefix(
req.params.docId, docWorkerMap, gristServer.getTag()
);
const info: PublicDocWorkerUrlInfo = selfPrefix ?
{ docWorkerUrl: null, selfPrefix } :
{ docWorkerUrl: customizeDocWorkerUrl(docWorker!.publicUrl, req), selfPrefix: null };
return res.json(info);
}));
// Handler for serving the document landing pages. Expects the following parameters:
@@ -160,7 +147,7 @@ export function attachAppEndpoint(options: AttachOptions): void {
// TODO docWorkerMain needs to serve app.html, perhaps with correct base-href already set.
const headers = {
Accept: 'application/json',
...getTransitiveHeaders(req),
...getTransitiveHeaders(req, { includeOrigin: true }),
};
const workerInfo = await getWorker(docWorkerMap, docId, `/${docId}/app.html`, {headers});
docStatus = workerInfo.docStatus;
@@ -206,10 +193,16 @@ export function attachAppEndpoint(options: AttachOptions): void {
});
}
// Without a public URL, we're in single server mode.
// Use a null workerPublicURL, to signify that the URL prefix serving the
// current endpoint is the only one available.
const publicUrl = docStatus?.docWorker?.publicUrl;
const workerPublicUrl = publicUrl !== undefined ? customizeDocWorkerUrl(publicUrl, req) : null;
await sendAppPage(req, res, {path: "", content: body.page, tag: body.tag, status: 200,
googleTagManager: 'anon', config: {
assignmentId: docId,
getWorker: {[docId]: customizeDocWorkerUrl(docStatus?.docWorker?.publicUrl, req)},
getWorker: {[docId]: workerPublicUrl },
getDoc: {[docId]: pruneAPIResult(doc as unknown as APIDocument)},
plugins
}});

8
app/server/lib/Authorizer.ts
View File

@@ -677,7 +677,10 @@ export function assertAccess(
* Pull out headers to pass along to a proxied service. Focused primarily on
* authentication.
*/
export function getTransitiveHeaders(req: Request): {[key: string]: string} {
export function getTransitiveHeaders(
req: Request,
{ includeOrigin }: { includeOrigin: boolean }
): {[key: string]: string} {
const Authorization = req.get('Authorization');
const Cookie = req.get('Cookie');
const PermitHeader = req.get('Permit');
@@ -685,13 +688,14 @@ export function getTransitiveHeaders(req: Request): {[key: string]: string} {
const XRequestedWith = req.get('X-Requested-With');
const Origin = req.get('Origin'); // Pass along the original Origin since it may
// play a role in granular access control.
const result: Record<string, string> = {
...(Authorization ? { Authorization } : undefined),
...(Cookie ? { Cookie } : undefined),
...(Organization ? { Organization } : undefined),
...(PermitHeader ? { Permit: PermitHeader } : undefined),
...(XRequestedWith ? { 'X-Requested-With': XRequestedWith } : undefined),
...(Origin ? { Origin } : undefined),
...((includeOrigin && Origin) ? { Origin } : undefined),
};
const extraHeader = process.env.GRIST_FORWARD_AUTH_HEADER;
const extraHeaderValue = extraHeader && req.get(extraHeader);

19
app/server/lib/BootProbes.ts
View File

@@ -58,6 +58,7 @@ export class BootProbes {
this._probes.push(_bootProbe);
this._probes.push(_hostHeaderProbe);
this._probes.push(_sandboxingProbe);
this._probes.push(_authenticationProbe);
this._probeById = new Map(this._probes.map(p => [p.id, p]));
}
}
@@ -77,7 +78,7 @@ const _homeUrlReachableProbe: Probe = {
id: 'reachable',
name: 'Grist is reachable',
apply: async (server, req) => {
const url = server.getHomeUrl(req);
const url = server.getHomeInternalUrl();
try {
const resp = await fetch(url);
if (resp.status !== 200) {
@@ -102,7 +103,7 @@ const _statusCheckProbe: Probe = {
id: 'health-check',
name: 'Built-in Health check',
apply: async (server, req) => {
const baseUrl = server.getHomeUrl(req);
const baseUrl = server.getHomeInternalUrl();
const url = new URL(baseUrl);
url.pathname = removeTrailingSlash(url.pathname) + '/status';
try {
@@ -202,3 +203,17 @@ const _sandboxingProbe: Probe = {
};
},
};
const _authenticationProbe: Probe = {
id: 'authentication',
name: 'Authentication system',
apply: async(server, req) => {
const loginSystemId = server.getInfo('loginMiddlewareComment');
return {
success: loginSystemId != undefined,
details: {
loginSystemId,
}
};
},
};

23
app/server/lib/DocApi.ts
View File

@@ -1098,10 +1098,11 @@ export class DocWorkerApi {
if (req.body.sourceDocId) {
options.sourceDocId = await this._confirmDocIdForRead(req, String(req.body.sourceDocId));
// Make sure that if we wanted to download the full source, we would be allowed.
const result = await fetch(this._grist.getHomeUrl(req, `/api/docs/${options.sourceDocId}/download?dryrun=1`), {
const homeUrl = this._grist.getHomeInternalUrl(`/api/docs/${options.sourceDocId}/download?dryrun=1`);
const result = await fetch(homeUrl, {
method: 'GET',
headers: {
...getTransitiveHeaders(req),
...getTransitiveHeaders(req, { includeOrigin: false }),
'Content-Type': 'application/json',
}
});
@@ -1111,10 +1112,10 @@ export class DocWorkerApi {
}
// We should make sure the source document has flushed recently.
// It may not be served by the same worker, so work through the api.
await fetch(this._grist.getHomeUrl(req, `/api/docs/${options.sourceDocId}/flush`), {
await fetch(this._grist.getHomeInternalUrl(`/api/docs/${options.sourceDocId}/flush`), {
method: 'POST',
headers: {
...getTransitiveHeaders(req),
...getTransitiveHeaders(req, { includeOrigin: false }),
'Content-Type': 'application/json',
}
});
@@ -1170,12 +1171,16 @@ export class DocWorkerApi {
const showDetails = isAffirmative(req.query.detail);
const docSession = docSessionFromRequest(req);
const {states} = await this._getStates(docSession, activeDoc);
const ref = await fetch(this._grist.getHomeUrl(req, `/api/docs/${req.params.docId2}/states`), {
const ref = await fetch(this._grist.getHomeInternalUrl(`/api/docs/${req.params.docId2}/states`), {
headers: {
...getTransitiveHeaders(req),
...getTransitiveHeaders(req, { includeOrigin: false }),
'Content-Type': 'application/json',
}
});
if (!ref.ok) {
res.status(ref.status).send(await ref.text());
return;
}
const states2: DocState[] = (await ref.json()).states;
const left = states[0];
const right = states2[0];
@@ -1199,9 +1204,9 @@ export class DocWorkerApi {
// Calculate changes from the (common) parent to the current version of the other document.
const url = `/api/docs/${req.params.docId2}/compare?left=${parent.h}`;
const rightChangesReq = await fetch(this._grist.getHomeUrl(req, url), {
const rightChangesReq = await fetch(this._grist.getHomeInternalUrl(url), {
headers: {
...getTransitiveHeaders(req),
...getTransitiveHeaders(req, { includeOrigin: false }),
'Content-Type': 'application/json',
}
});
@@ -1644,7 +1649,7 @@ export class DocWorkerApi {
let uploadResult;
try {
const accessId = makeAccessId(req, getAuthorizedUserId(req));
uploadResult = await fetchDoc(this._grist, sourceDocumentId, req, accessId, asTemplate);
uploadResult = await fetchDoc(this._grist, this._docWorkerMap, sourceDocumentId, req, accessId, asTemplate);
globalUploadSet.changeUploadName(uploadResult.uploadId, accessId, `${documentName}.grist`);
} catch (err) {
if ((err as ApiError).status === 403) {

51
app/server/lib/DocWorkerUtils.ts
View File

@@ -1,11 +1,12 @@
import {ApiError} from 'app/common/ApiError';
import {parseSubdomainStrictly} from 'app/common/gristUrls';
import {removeTrailingSlash} from 'app/common/gutil';
import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import log from 'app/server/lib/log';
import {adaptServerUrl} from 'app/server/lib/requestUtils';
import * as express from 'express';
import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
import {getAssignmentId} from './idUtils';
/**
* This method transforms a doc worker's public url as needed based on the request.
@@ -35,16 +36,7 @@ import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
* TODO: doc worker registration could be redesigned to remove the assumption
* of a fixed base domain.
*/
export function customizeDocWorkerUrl(
docWorkerUrlSeed: string|undefined,
req: express.Request
): string|null {
if (!docWorkerUrlSeed) {
// When no doc worker seed, we're in single server mode.
// Return null, to signify that the URL prefix serving the
// current endpoint is the only one available.
return null;
}
export function customizeDocWorkerUrl( docWorkerUrlSeed: string, req: express.Request): string {
const docWorkerUrl = new URL(docWorkerUrlSeed);
const workerSubdomain = parseSubdomainStrictly(docWorkerUrl.hostname).org;
adaptServerUrl(docWorkerUrl, req);
@@ -152,6 +144,43 @@ export async function getWorker(
}
}
export type DocWorkerInfoOrSelfPrefix = {
docWorker: DocWorkerInfo,
selfPrefix?: never,
} | {
docWorker?: never,
selfPrefix: string
};
export async function getDocWorkerInfoOrSelfPrefix(
docId: string,
docWorkerMap?: IDocWorkerMap | null,
tag?: string
): Promise<DocWorkerInfoOrSelfPrefix> {
if (!useWorkerPool()) {
// Let the client know there is not a separate pool of workers,
// so they should continue to use the same base URL for accessing
// documents. For consistency, return a prefix to add into that
// URL, as there would be for a pool of workers. It would be nice
// to go ahead and provide the full URL, but that requires making
// more assumptions about how Grist is configured.
// Alternatives could be: have the client to send their base URL
// in the request; or use headers commonly added by reverse proxies.
const selfPrefix = "/dw/self/v/" + tag;
return { selfPrefix };
}
if (!docWorkerMap) {
throw new Error('no worker map');
}
const assignmentId = getAssignmentId(docWorkerMap, docId);
const { docStatus } = await getWorker(docWorkerMap, assignmentId, '/status');
if (!docStatus) {
throw new Error('no worker');
}
return { docWorker: docStatus.docWorker };
}
// Return true if document related endpoints are served by separate workers.
export function useWorkerPool() {
return process.env.GRIST_SINGLE_PORT !== 'true';

36
app/server/lib/FlexServer.ts
View File

@@ -295,6 +295,13 @@ export class FlexServer implements GristServer {
return homeUrl;
}
/**
* Same as getDefaultHomeUrl, but for internal use.
*/
public getDefaultHomeInternalUrl(): string {
return process.env.APP_HOME_INTERNAL_URL || this.getDefaultHomeUrl();
}
/**
* Get a url for the home server api, adapting it to match the base domain in the
* requested url. This adaptation is important for cookie-based authentication.
@@ -309,6 +316,14 @@ export class FlexServer implements GristServer {
return homeUrl.href;
}
/**
* Same as getHomeUrl, but for requesting internally.
*/
public getHomeInternalUrl(relPath: string = ''): string {
const homeUrl = new URL(relPath, this.getDefaultHomeInternalUrl());
return homeUrl.href;
}
/**
* Get a home url that is appropriate for the given document. For now, this
* returns a default that works for all documents. That could change in future,
@@ -316,7 +331,7 @@ export class FlexServer implements GristServer {
* based on domain).
*/
public async getHomeUrlByDocId(docId: string, relPath: string = ''): Promise<string> {
return new URL(relPath, this.getDefaultHomeUrl()).href;
return new URL(relPath, this.getDefaultHomeInternalUrl()).href;
}
// Get the port number the server listens on. This may be different from the port
@@ -1411,6 +1426,11 @@ export class FlexServer implements GristServer {
return this._sandboxInfo;
}
public getInfo(key: string): any {
const infoPair = this.info.find(([keyToCheck]) => key === keyToCheck);
return infoPair?.[1];
}
public disableExternalStorage() {
if (this.deps.has('doc')) {
throw new Error('disableExternalStorage called too late');
@@ -1429,12 +1449,12 @@ export class FlexServer implements GristServer {
return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}});
}));
const createDoom = async (req: express.Request) => {
const createDoom = async () => {
const dbManager = this.getHomeDBManager();
const permitStore = this.getPermitStore();
const notifier = this.getNotifier();
const loginSystem = await this.resolveLoginSystem();
const homeUrl = this.getHomeUrl(req).replace(/\/$/, '');
const homeUrl = this.getHomeInternalUrl().replace(/\/$/, '');
return new Doom(dbManager, permitStore, notifier, loginSystem, homeUrl);
};
@@ -1458,7 +1478,7 @@ export class FlexServer implements GristServer {
// Reuse Doom cli tool for account deletion. It won't allow to delete account if it has access
// to other (not public) team sites.
const doom = await createDoom(req);
const doom = await createDoom();
await doom.deleteUser(userId);
this.getTelemetry().logEvent(req as RequestWithLogin, 'deletedAccount');
return resp.status(200).json(true);
@@ -1491,7 +1511,7 @@ export class FlexServer implements GristServer {
}
// Reuse Doom cli tool for org deletion. Note, this removes everything as a super user.
const doom = await createDoom(req);
const doom = await createDoom();
await doom.deleteOrg(org.id);
this.getTelemetry().logEvent(req as RequestWithLogin, 'deletedSite', {
@@ -1980,7 +2000,7 @@ export class FlexServer implements GristServer {
// Add the handling for the /upload route. Most uploads are meant for a DocWorker: they are put
// in temporary files, and the DocWorker needs to be on the same machine to have access to them.
// This doesn't check for doc access permissions because the request isn't tied to a document.
addUploadRoute(this, this.app, this._trustOriginsMiddleware, ...basicMiddleware);
addUploadRoute(this, this.app, this._docWorkerMap, this._trustOriginsMiddleware, ...basicMiddleware);
this.app.get('/attachment', ...docAccessMiddleware,
expressWrap(async (req, res) => this._docWorker.getAttachment(req, res)));
@@ -2418,10 +2438,10 @@ export class FlexServer implements GristServer {
const workspace = workspaces.find(w => w.name === 'Home');
if (!workspace) { throw new Error('Home workspace not found'); }
const copyDocUrl = this.getHomeUrl(req, '/api/docs');
const copyDocUrl = this.getHomeInternalUrl('/api/docs');
const response = await fetch(copyDocUrl, {
headers: {
...getTransitiveHeaders(req),
...getTransitiveHeaders(req, { includeOrigin: false }),
'Content-Type': 'application/json',
},
method: 'POST',

4
app/server/lib/GristServer.ts
View File

@@ -35,6 +35,7 @@ export interface GristServer {
settings?: Readonly<Record<string, unknown>>;
getHost(): string;
getHomeUrl(req: express.Request, relPath?: string): string;
getHomeInternalUrl(relPath?: string): string;
getHomeUrlByDocId(docId: string, relPath?: string): Promise<string>;
getOwnUrl(): string;
getOrgUrl(orgKey: string|number): Promise<string>;
@@ -66,6 +67,7 @@ export interface GristServer {
getBundledWidgets(): ICustomWidget[];
hasBoot(): boolean;
getSandboxInfo(): SandboxInfo|undefined;
getInfo(key: string): any;
}
export interface GristLoginSystem {
@@ -127,6 +129,7 @@ export function createDummyGristServer(): GristServer {
settings: {},
getHost() { return 'localhost:4242'; },
getHomeUrl() { return 'http://localhost:4242'; },
getHomeInternalUrl() { return 'http://localhost:4242'; },
getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); },
getMergedOrgUrl() { return 'http://localhost:4242'; },
getOwnUrl() { return 'http://localhost:4242'; },
@@ -157,6 +160,7 @@ export function createDummyGristServer(): GristServer {
getBundledWidgets() { return []; },
hasBoot() { return false; },
getSandboxInfo() { return undefined; },
getInfo(key: string) { return undefined; }
};
}

1
app/server/lib/ITestingHooks-ti.ts
View File

@@ -11,7 +11,6 @@ export const ClientJsonMemoryLimits = t.iface([], {
});
export const ITestingHooks = t.iface([], {
"getOwnPort": t.func("number"),
"getPort": t.func("number"),
"setLoginSessionProfile": t.func("void", t.param("gristSidCookie", "string"), t.param("profile", t.union("UserProfile", "null")), t.param("org", "string", true)),
"setServerVersion": t.func("void", t.param("version", t.union("string", "null"))),

1
app/server/lib/ITestingHooks.ts
View File

@@ -7,7 +7,6 @@ export interface ClientJsonMemoryLimits {
}
export interface ITestingHooks {
getOwnPort(): Promise<number>;
getPort(): Promise<number>;
setLoginSessionProfile(gristSidCookie: string, profile: UserProfile|null, org?: string): Promise<void>;
setServerVersion(version: string|null): Promise<void>;

5
app/server/lib/TestingHooks.ts
View File

@@ -68,11 +68,6 @@ export class TestingHooks implements ITestingHooks {
private _workerServers: FlexServer[]
) {}
public async getOwnPort(): Promise<number> {
log.info("TestingHooks.getOwnPort called");
return this._server.getOwnPort();
}
public async getPort(): Promise<number> {
log.info("TestingHooks.getPort called");
return this._port;

2
app/server/lib/requestUtils.ts
View File

@@ -1,5 +1,5 @@
import {ApiError} from 'app/common/ApiError';
import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
import { DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail } from 'app/common/gristUrls';
import * as gutil from 'app/common/gutil';
import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager';
import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';

2
app/server/lib/sendAppPage.ts
View File

@@ -4,6 +4,7 @@ import {
getFreeCoachingCallUrl,
getHelpCenterUrl,
getPageTitleSuffix,
getTermsOfServiceUrl,
GristLoadConfig,
IFeature
} from 'app/common/gristUrls';
@@ -62,6 +63,7 @@ export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfi
baseDomain,
singleOrg: process.env.GRIST_SINGLE_ORG,
helpCenterUrl: getHelpCenterUrl(),
termsOfServiceUrl: getTermsOfServiceUrl(),
freeCoachingCallUrl: getFreeCoachingCallUrl(),
contactSupportUrl: getContactSupportUrl(),
pathOnly,

38
app/server/lib/uploads.ts
View File

@@ -1,7 +1,7 @@
import {ApiError} from 'app/common/ApiError';
import {InactivityTimer} from 'app/common/InactivityTimer';
import {FetchUrlOptions, FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
import {getDocWorkerUrl} from 'app/common/UserAPI';
import {getUrlFromPrefix} from 'app/common/UserAPI';
import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
RequestWithLogin} from 'app/server/lib/Authorizer';
import {expressWrap} from 'app/server/lib/expressWrap';
@@ -21,6 +21,8 @@ import * as multiparty from 'multiparty';
import fetch, {Response as FetchResponse} from 'node-fetch';
import * as path from 'path';
import * as tmp from 'tmp';
import {IDocWorkerMap} from './DocWorkerMap';
import {getDocWorkerInfoOrSelfPrefix} from './DocWorkerUtils';
// After some time of inactivity, clean up the upload. We give an hour, which seems generous,
// except that if one is toying with import options, and leaves the upload in an open browser idle
@@ -39,7 +41,12 @@ export interface FormResult {
/**
* Adds an upload route to the given express app, listening for POST requests at UPLOAD_URL_PATH.
*/
export function addUploadRoute(server: GristServer, expressApp: Application, ...handlers: RequestHandler[]): void {
export function addUploadRoute(
server: GristServer,
expressApp: Application,
docWorkerMap: IDocWorkerMap,
...handlers: RequestHandler[]
): void {
// When doing a cross-origin post, the browser will check for access with options prior to posting.
// We need to reassure it that the request will be accepted before it will go ahead and post.
@@ -72,7 +79,7 @@ export function addUploadRoute(server: GristServer, expressApp: Application, ...
if (!docId) { throw new Error('doc must be specified'); }
const accessId = makeAccessId(req, getAuthorizedUserId(req));
try {
const uploadResult: UploadResult = await fetchDoc(server, docId, req, accessId,
const uploadResult: UploadResult = await fetchDoc(server, docWorkerMap, docId, req, accessId,
req.query.template === '1');
if (name) {
globalUploadSet.changeUploadName(uploadResult.uploadId, accessId, name);
@@ -404,24 +411,21 @@ async function _fetchURL(url: string, accessId: string|null, options?: FetchUrlO
* Fetches a Grist doc potentially managed by a different doc worker. Passes on credentials
* supplied in the current request.
*/
export async function fetchDoc(server: GristServer, docId: string, req: Request, accessId: string|null,
template: boolean): Promise<UploadResult> {
export async function fetchDoc(
server: GristServer,
docWorkerMap: IDocWorkerMap,
docId: string,
req: Request,
accessId: string|null,
template: boolean
): Promise<UploadResult> {
// Prepare headers that preserve credentials of current user.
const headers = getTransitiveHeaders(req);
// Passing the Origin header would serve no purpose here, as we are
// constructing an internal request to fetch from our own doc worker
// URL. Indeed, it may interfere, as it could incur a CORS check in
// `trustOrigin`, which we do not need.
delete headers.Origin;
const headers = getTransitiveHeaders(req, { includeOrigin: false });
// Find the doc worker responsible for the document we wish to copy.
// The backend needs to be well configured for this to work.
const homeUrl = server.getHomeUrl(req);
const fetchUrl = new URL(`/api/worker/${docId}`, homeUrl);
const response: FetchResponse = await Deps.fetch(fetchUrl.href, {headers});
await _checkForError(response);
const docWorkerUrl = getDocWorkerUrl(server.getOwnUrl(), await response.json());
const { selfPrefix, docWorker } = await getDocWorkerInfoOrSelfPrefix(docId, docWorkerMap, server.getTag());
const docWorkerUrl = docWorker ? docWorker.internalUrl : getUrlFromPrefix(server.getHomeInternalUrl(), selfPrefix);
// Download the document, in full or as a template.
const url = new URL(`api/docs/${docId}/download?template=${Number(template)}`,
docWorkerUrl.replace(/\/*$/, '/'));