Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

Make changes required for Desktop FS updates (#1099)

Browse Source 
Make a set of changes required for Desktop FS improvements, see
https://github.com/gristlabs/grist-desktop/pull/42

---------

Co-authored-by: Spoffy <contact@spoffy.net>
Co-authored-by: Spoffy <4805393+Spoffy@users.noreply.github.com>
This commit is contained in:
Leslie H 2024-09-17 01:01:58 +00:00 committed by GitHub
parent 938bb0666e
commit 02cfcee84d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 552 additions and 447 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.gitignore vendored
View File

@ -12,6 +12,11 @@
/sandbox_venv*
/.vscode/
# Files created by grist-desktop setup
/cpython.tar.gz
/python
/static_ext
# Build helper files.
/.build*
@ -82,7 +87,8 @@ xunit.xml
**/_build
# ext directory can be overwritten
ext/**
/ext
/ext/**
# Docker compose examples - persistent values and secrets
/docker-compose-examples/*/persist

66
app/client/ui/HomeImports.ts → app/client/ui/CoreHomeImports.ts
View File

@ -1,13 +1,13 @@
import {AppModel, reportError} from 'app/client/models/AppModel';
import {AxiosProgressEvent} from 'axios';
import {PluginScreen} from 'app/client/components/PluginScreen';
import {guessTimezone} from 'app/client/lib/guessTimezone';
import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
import {IMPORTABLE_EXTENSIONS, uploadFiles} from 'app/client/lib/uploads';
import {AppModel, reportError} from 'app/client/models/AppModel';
import {IProgress} from 'app/client/models/NotifyModel';
import {ImportProgress} from 'app/client/ui/ImportProgress';
import {IMPORTABLE_EXTENSIONS} from 'app/client/lib/uploads';
import {openFilePicker} from 'app/client/ui/FileDialog';
import {byteString} from 'app/common/gutil';
import { AxiosProgressEvent } from 'axios';
import {Disposable} from 'grainjs';
import {uploadFiles} from 'app/client/lib/uploads';
/**
* Imports a document and returns its docId, or null if no files were selected.
@ -66,62 +66,6 @@ export async function fileImport(
progressUI.dispose();
}
}
export class ImportProgress extends Disposable {
// Import does upload first, then import. We show a single indicator, estimating which fraction
// of the time should be given to upload (whose progress we can report well), and which to the
// subsequent import (whose progress indicator is mostly faked).
private _uploadFraction: number;
private _estImportSeconds: number;
private _importTimer: null | ReturnType<typeof setInterval> = null;
private _importStart: number = 0;
constructor(private _progressUI: IProgress, file: File) {
super();
// We'll assume that for .grist files, the upload takes 90% of the total time, and for other
// files, 40%.
this._uploadFraction = file.name.endsWith(".grist") ? 0.9 : 0.4;
// TODO: Import step should include a progress callback, to be combined with upload progress.
// Without it, we estimate import to take 2s per MB (non-scientific unreliable estimate), and
// use an asymptotic indicator which keeps moving without ever finishing. Not terribly useful,
// but does slow down for larger files, and is more comforting than a stuck indicator.
this._estImportSeconds = file.size / 1024 / 1024 * 2;
this._progressUI.setProgress(0);
this.onDispose(() => this._importTimer && clearInterval(this._importTimer));
}
// Once this reaches 100, the import stage begins.
public setUploadProgress(percentage: number) {
this._progressUI.setProgress(percentage * this._uploadFraction);
if (percentage >= 100 && !this._importTimer) {
this._importStart = Date.now();
this._importTimer = setInterval(() => this._onImportTimer(), 100);
}
}
public finish() {
if (this._importTimer) {
clearInterval(this._importTimer);
}
this._progressUI.setProgress(100);
}
/**
* Calls _progressUI.setProgress(percent) with percentage increasing from 0 and asymptotically
* approaching 100, reaching 50% after estSeconds. It's intended to look reasonable when the
* estimate is good, and to keep showing slowing progress even if it's not.
*/
private _onImportTimer() {
const elapsedSeconds = (Date.now() - this._importStart) / 1000;
const importProgress = elapsedSeconds / (elapsedSeconds + this._estImportSeconds);
const progress = this._uploadFraction + importProgress * (1 - this._uploadFraction);
this._progressUI.setProgress(100 * progress);
}
}
/**
* Imports document through a plugin from a home/welcome screen.
*/

47
app/client/ui/CoreNewDocMethods.ts Normal file
View File

@ -0,0 +1,47 @@
import {homeImports} from 'app/client/ui/HomeImports';
import {docUrl, urlState} from 'app/client/models/gristUrlState';
import {HomeModel} from 'app/client/models/HomeModel';
import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
import {reportError} from 'app/client/models/AppModel';
export async function createDocAndOpen(home: HomeModel) {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
try {
const docId = await home.createDoc("Untitled document", destWS === "unsaved" ? "unsaved" : destWS.id);
// Fetch doc information including urlId.
// TODO: consider changing API to return same response as a GET when creating an
// object, which is a semi-standard.
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
} catch (err) {
reportError(err);
}
}
export async function importDocAndOpen(home: HomeModel) {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
const docId = await homeImports.docImport(home.app, destWS === "unsaved" ? "unsaved" : destWS.id);
if (docId) {
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
}
}
export async function importFromPluginAndOpen(home: HomeModel, source: ImportSourceElement) {
try {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
const docId = await homeImports.importFromPlugin(
home.app,
destWS === "unsaved" ? "unsaved" : destWS.id,
source);
if (docId) {
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
}
} catch (err) {
reportError(err);
}
}

6
app/client/ui/HomeIntro.ts
View File

@ -3,7 +3,7 @@ import {getLoginOrSignupUrl, getLoginUrl, getSignupUrl, urlState} from 'app/clie
import {HomeModel} from 'app/client/models/HomeModel';
import {productPill} from 'app/client/ui/AppHeader';
import * as css from 'app/client/ui/DocMenuCss';
import {createDocAndOpen, importDocAndOpen} from 'app/client/ui/HomeLeftPane';
import {newDocMethods} from 'app/client/ui/NewDocMethods';
import {manageTeamUsersApp} from 'app/client/ui/OpenUserManager';
import {bigBasicButton, cssButton} from 'app/client/ui2018/buttons';
import {testId, theme, vars} from 'app/client/ui2018/cssVars';
@ -177,11 +177,11 @@ function buildButtons(homeModel: HomeModel, options: {
),
!options.import ? null :
cssBtn(cssBtnIcon('Import'), t("Import Document"), testId('intro-import-doc'),
dom.on('click', () => importDocAndOpen(homeModel)),
dom.on('click', () => newDocMethods.importDocAndOpen(homeModel)),
),
!options.empty ? null :
cssBtn(cssBtnIcon('Page'), t("Create Empty Document"), testId('intro-create-doc'),
dom.on('click', () => createDocAndOpen(homeModel)),
dom.on('click', () => newDocMethods.createDocAndOpen(homeModel)),
),
);
}

72
app/client/ui/HomeLeftPane.ts
View File

@ -1,29 +1,27 @@
import {makeT} from 'app/client/lib/localization';
import {loadUserManager} from 'app/client/lib/imports';
import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
import {reportError} from 'app/client/models/AppModel';
import {docUrl, urlState} from 'app/client/models/gristUrlState';
import {urlState} from 'app/client/models/gristUrlState';
import {HomeModel} from 'app/client/models/HomeModel';
import {getWorkspaceInfo, workspaceName} from 'app/client/models/WorkspaceInfo';
import {getAdminPanelName} from 'app/client/ui/AdminPanelName';
import * as roles from 'app/common/roles';
import {addNewButton, cssAddNewButton} from 'app/client/ui/AddNewButton';
import {docImport, importFromPlugin} from 'app/client/ui/HomeImports';
import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls';
import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs';
import {newDocMethods} from 'app/client/ui/NewDocMethods';
import {createHelpTools, cssLeftPanel, cssScrollPane,
cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon';
import {
cssLinkText, cssMenuTrigger, cssPageEntry, cssPageIcon, cssPageLink, cssSpacer
} from 'app/client/ui/LeftPanelCommon';
import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour';
import {transientInput} from 'app/client/ui/transientInput';
import {testId, theme} from 'app/client/ui2018/cssVars';
import {icon} from 'app/client/ui2018/icons';
import {menu, menuIcon, menuItem, upgradableMenuItem, upgradeText} from 'app/client/ui2018/menus';
import {testId, theme} from 'app/client/ui2018/cssVars';
import {confirmModal} from 'app/client/ui2018/modals';
import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls';
import * as roles from 'app/common/roles';
import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour';
import {getGristConfig} from 'app/common/urlUtils';
import {icon} from 'app/client/ui2018/icons';
import {transientInput} from 'app/client/ui/transientInput';
import {Workspace} from 'app/common/UserAPI';
import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs';
import {createHelpTools, cssLeftPanel, cssScrollPane,
cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon';
const t = makeT('HomeLeftPane');
@ -160,65 +158,23 @@ export function createHomeLeftPane(leftPanelOpen: Observable<boolean>, home: Hom
);
}
export async function createDocAndOpen(home: HomeModel) {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
try {
const docId = await home.createDoc("Untitled document", destWS === "unsaved" ? "unsaved" : destWS.id);
// Fetch doc information including urlId.
// TODO: consider changing API to return same response as a GET when creating an
// object, which is a semi-standard.
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
} catch (err) {
reportError(err);
}
}
export async function importDocAndOpen(home: HomeModel) {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
const docId = await docImport(home.app, destWS === "unsaved" ? "unsaved" : destWS.id);
if (docId) {
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
}
}
export async function importFromPluginAndOpen(home: HomeModel, source: ImportSourceElement) {
try {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
const docId = await importFromPlugin(
home.app,
destWS === "unsaved" ? "unsaved" : destWS.id,
source);
if (docId) {
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
}
} catch (err) {
reportError(err);
}
}
function addMenu(home: HomeModel, creating: Observable<boolean>): DomElementArg[] {
const org = home.app.currentOrg;
const orgAccess: roles.Role|null = org ? org.access : null;
const needUpgrade = home.app.currentFeatures?.maxWorkspacesPerOrg === 1;
return [
menuItem(() => createDocAndOpen(home), menuIcon('Page'), t("Create Empty Document"),
menuItem(() => newDocMethods.createDocAndOpen(home), menuIcon('Page'), t("Create Empty Document"),
dom.cls('disabled', !home.newDocWorkspace.get()),
testId("dm-new-doc")
),
menuItem(() => importDocAndOpen(home), menuIcon('Import'), t("Import Document"),
menuItem(() => newDocMethods.importDocAndOpen(home), menuIcon('Import'), t("Import Document"),
dom.cls('disabled', !home.newDocWorkspace.get()),
testId("dm-import")
),
domComputed(home.importSources, importSources => ([
...importSources.map((source, i) =>
menuItem(() => importFromPluginAndOpen(home, source),
menuItem(() => newDocMethods.importFromPluginAndOpen(home, source),
menuIcon('Import'),
source.importSource.label,
dom.cls('disabled', !home.newDocWorkspace.get()),

58
app/client/ui/ImportProgress.ts Normal file
View File

@ -0,0 +1,58 @@
import {IProgress} from 'app/client/models/NotifyModel';
import {Disposable} from 'grainjs';
export class ImportProgress extends Disposable {
// Import does upload first, then import. We show a single indicator, estimating which fraction
// of the time should be given to upload (whose progress we can report well), and which to the
// subsequent import (whose progress indicator is mostly faked).
private _uploadFraction: number;
private _estImportSeconds: number;
private _importTimer: null | ReturnType<typeof setInterval> = null;
private _importStart: number = 0;
constructor(private _progressUI: IProgress, file: File) {
super();
// We'll assume that for .grist files, the upload takes 90% of the total time, and for other
// files, 40%.
this._uploadFraction = file.name.endsWith(".grist") ? 0.9 : 0.4;
// TODO: Import step should include a progress callback, to be combined with upload progress.
// Without it, we estimate import to take 2s per MB (non-scientific unreliable estimate), and
// use an asymptotic indicator which keeps moving without ever finishing. Not terribly useful,
// but does slow down for larger files, and is more comforting than a stuck indicator.
this._estImportSeconds = file.size / 1024 / 1024 * 2;
this._progressUI.setProgress(0);
this.onDispose(() => this._importTimer && clearInterval(this._importTimer));
}
// Once this reaches 100, the import stage begins.
public setUploadProgress(percentage: number) {
this._progressUI.setProgress(percentage * this._uploadFraction);
if (percentage >= 100 && !this._importTimer) {
this._importStart = Date.now();
this._importTimer = setInterval(() => this._onImportTimer(), 100);
}
}
public finish() {
if (this._importTimer) {
clearInterval(this._importTimer);
}
this._progressUI.setProgress(100);
}
/**
* Calls _progressUI.setProgress(percent) with percentage increasing from 0 and asymptotically
* approaching 100, reaching 50% after estSeconds. It's intended to look reasonable when the
* estimate is good, and to keep showing slowing progress even if it's not.
*/
private _onImportTimer() {
const elapsedSeconds = (Date.now() - this._importStart) / 1000;
const importProgress = elapsedSeconds / (elapsedSeconds + this._estImportSeconds);
const progress = this._uploadFraction + importProgress * (1 - this._uploadFraction);
this._progressUI.setProgress(100 * progress);
}
}

16
app/gen-server/lib/homedb/HomeDBManager.ts
View File

@ -68,12 +68,12 @@ import {Request} from "express";
import {defaultsDeep, flatten, pick} from 'lodash';
import {
Brackets,
Connection,
DatabaseType,
DataSource,
EntityManager,
ObjectLiteral,
SelectQueryBuilder,
WhereExpression
WhereExpressionBuilder
} from "typeorm";
import uuidv4 from "uuid/v4";
@ -247,7 +247,7 @@ export type BillingOptions = Partial<Pick<BillingAccount,
*/
export class HomeDBManager extends EventEmitter {
private _usersManager = new UsersManager(this, this._runInTransaction.bind(this));
private _connection: Connection;
private _connection: DataSource;
private _exampleWorkspaceId: number;
private _exampleOrgId: number;
private _idPrefix: string = ""; // Place this before ids in subdomains, used in routing to
@ -353,7 +353,7 @@ export class HomeDBManager extends EventEmitter {
this._connection = await getOrCreateConnection();
}
public connectTo(connection: Connection) {
public connectTo(connection: DataSource) {
this._connection = connection;
}
@ -987,6 +987,10 @@ export class HomeDBManager extends EventEmitter {
return doc;
}
public async getAllDocs() {
return this.connection.getRepository(Document).find();
}
public async getRawDocById(docId: string, transaction?: EntityManager) {
return await this.getDoc({
urlId: docId,
@ -3438,7 +3442,7 @@ export class HomeDBManager extends EventEmitter {
// Adds a where clause to filter orgs by domain or id.
// If org is null, filter for user's personal org.
// if includeSupport is true, include the org of the support@ user (for the Samples workspace)
private _whereOrg<T extends WhereExpression>(qb: T, org: string|number, includeSupport = false): T {
private _whereOrg<T extends WhereExpressionBuilder>(qb: T, org: string|number, includeSupport = false): T {
if (this.isMergedOrg(org)) {
// Select from universe of personal orgs.
// Don't panic though! While this means that SQL can't use an organization id
@ -3458,7 +3462,7 @@ export class HomeDBManager extends EventEmitter {
}
}
private _wherePlainOrg<T extends WhereExpression>(qb: T, org: string|number): T {
private _wherePlainOrg<T extends WhereExpressionBuilder>(qb: T, org: string|number): T {
if (typeof org === 'number') {
return qb.andWhere('orgs.id = :org', {org});
}

237
app/server/MergedServer.ts Normal file
View File

@ -0,0 +1,237 @@
/**
*
* A version of hosted grist that recombines a home server,
* a doc worker, and a static server on a single port.
*
*/
import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
import log from 'app/server/lib/log';
import {getGlobalConfig} from "app/server/lib/globalConfig";
// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
// environment variable.
export type ServerType = "home" | "docs" | "static" | "app";
const allServerTypes: ServerType[] = ["home", "docs", "static", "app"];
// Parse a comma-separate list of server types into an array, with validation.
export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
// Split and filter out empty strings (including the one we get when splitting "").
const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part));
// Check that parts is non-empty and only contains valid options.
if (!types.length) {
throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`);
}
for (const t of types) {
if (!allServerTypes.includes(t as ServerType)) {
throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`);
}
}
return types as ServerType[];
}
function checkUserContentPort(): number | null {
// Check whether a port is explicitly set for user content.
if (process.env.GRIST_UNTRUSTED_PORT) {
return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
}
// Checks whether to serve user content on same domain but on different port
if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
const homeUrl = new URL(process.env.APP_HOME_URL);
const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
// If the hostname of both home and plugin url are the same,
// but the ports are different
if (homeUrl.hostname === pluginUrl.hostname &&
homeUrl.port !== pluginUrl.port) {
const port = parseInt(pluginUrl.port || '80', 10);
return port;
}
}
return null;
}
interface ServerOptions extends FlexServerOptions {
// If set, messages logged to console (default: false)
// (but if options are not given at all in call to main, logToConsole is set to true)
logToConsole?: boolean;
// If set, documents saved to external storage such as s3 (default is to check environment variables,
// which get set in various ways in dev/test entry points)
externalStorage?: boolean;
}
export class MergedServer {
public static async create(port: number, serverTypes: ServerType[], options: ServerOptions = {}) {
options.settings ??= getGlobalConfig();
const ms = new MergedServer(port, serverTypes, options);
// We need to know early on whether we will be serving plugins or not.
if (ms.hasComponent("home")) {
const userPort = checkUserContentPort();
ms.flexServer.setServesPlugins(userPort !== undefined);
} else {
ms.flexServer.setServesPlugins(false);
}
ms.flexServer.addCleanup();
ms.flexServer.setDirectory();
if (process.env.GRIST_TEST_ROUTER) {
// Add a mock api for adding/removing doc workers from load balancer.
ms.flexServer.testAddRouter();
}
if (ms._options.logToConsole !== false) { ms.flexServer.addLogging(); }
if (ms._options.externalStorage === false) { ms.flexServer.disableExternalStorage(); }
await ms.flexServer.addLoginMiddleware();
if (ms.hasComponent("docs")) {
// It is important that /dw and /v prefixes are accepted (if present) by health check
// in ms case, since they are included in the url registered for the doc worker.
ms.flexServer.stripDocWorkerIdPathPrefixIfPresent();
ms.flexServer.addTagChecker();
}
ms.flexServer.addHealthCheck();
if (ms.hasComponent("home") || ms.hasComponent("app")) {
ms.flexServer.addBootPage();
}
ms.flexServer.denyRequestsIfNotReady();
if (ms.hasComponent("home") || ms.hasComponent("static") || ms.hasComponent("app")) {
ms.flexServer.setDirectory();
}
if (ms.hasComponent("home") || ms.hasComponent("static")) {
ms.flexServer.addStaticAndBowerDirectories();
}
await ms.flexServer.initHomeDBManager();
ms.flexServer.addHosts();
ms.flexServer.addDocWorkerMap();
if (ms.hasComponent("home") || ms.hasComponent("static")) {
await ms.flexServer.addAssetsForPlugins();
}
if (ms.hasComponent("home")) {
ms.flexServer.addEarlyWebhooks();
}
if (ms.hasComponent("home") || ms.hasComponent("docs") || ms.hasComponent("app")) {
ms.flexServer.addSessions();
}
ms.flexServer.addAccessMiddleware();
ms.flexServer.addApiMiddleware();
await ms.flexServer.addBillingMiddleware();
return ms;
}
public readonly flexServer: FlexServer;
private readonly _serverTypes: ServerType[];
private readonly _options: ServerOptions;
private constructor(port: number, serverTypes: ServerType[], options: ServerOptions = {}) {
this._serverTypes = serverTypes;
this._options = options;
this.flexServer = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
}
public hasComponent(serverType: ServerType) {
return this._serverTypes.includes(serverType);
}
public async run() {
try {
await this.flexServer.start();
if (this.hasComponent("home")) {
this.flexServer.addUsage();
if (!this.hasComponent("docs")) {
this.flexServer.addDocApiForwarder();
}
this.flexServer.addJsonSupport();
this.flexServer.addUpdatesCheck();
await this.flexServer.addLandingPages();
// todo: add support for home api to standalone app
this.flexServer.addHomeApi();
this.flexServer.addBillingApi();
this.flexServer.addNotifier();
this.flexServer.addAuditLogger();
await this.flexServer.addTelemetry();
await this.flexServer.addHousekeeper();
await this.flexServer.addLoginRoutes();
this.flexServer.addAccountPage();
this.flexServer.addBillingPages();
this.flexServer.addWelcomePaths();
this.flexServer.addLogEndpoint();
this.flexServer.addGoogleAuthEndpoint();
this.flexServer.addInstallEndpoints();
this.flexServer.addConfigEndpoints();
}
if (this.hasComponent("docs")) {
this.flexServer.addJsonSupport();
this.flexServer.addAuditLogger();
await this.flexServer.addTelemetry();
await this.flexServer.addDoc();
}
if (this.hasComponent("home")) {
this.flexServer.addClientSecrets();
}
this.flexServer.finalizeEndpoints();
await this.flexServer.finalizePlugins(this.hasComponent("home") ? checkUserContentPort() : null);
this.flexServer.checkOptionCombinations();
this.flexServer.summary();
this.flexServer.ready();
// Some tests have their timing perturbed by having this earlier
// TODO: update those tests.
if (this.hasComponent("docs")) {
await this.flexServer.checkSandbox();
}
} catch(e) {
await this.flexServer.close();
throw e;
}
}
}
export async function startMain() {
try {
const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
// No defaults for a port, since this server can serve very different purposes.
if (!process.env.GRIST_PORT) {
throw new Error("GRIST_PORT must be specified");
}
const port = parseInt(process.env.GRIST_PORT, 10);
const server = await MergedServer.create(port, serverTypes);
await server.run();
const opt = process.argv[2];
if (opt === '--testingHooks') {
await server.flexServer.addTestingHooks();
}
return server.flexServer;
} catch (e) {
log.error('mergedServer failed to start', e);
process.exit(1);
}
}
if (require.main === module) {
startMain().catch((e) => log.error('mergedServer failed to start', e));
}

18
app/server/devServerMain.ts
View File

@ -21,7 +21,7 @@
import {updateDb} from 'app/server/lib/dbUtils';
import {FlexServer} from 'app/server/lib/FlexServer';
import log from 'app/server/lib/log';
import {main as mergedServerMain} from 'app/server/mergedServerMain';
import {MergedServer} from 'app/server/MergedServer';
import {promisifyAll} from 'bluebird';
import * as fse from 'fs-extra';
import * as path from 'path';
@ -96,8 +96,9 @@ export async function main() {
if (!process.env.APP_HOME_URL) {
process.env.APP_HOME_URL = `http://localhost:${port}`;
}
const server = await mergedServerMain(port, ["home", "docs", "static"]);
await server.addTestingHooks();
const mergedServer = await MergedServer.create(port, ["home", "docs", "static"]);
await mergedServer.flexServer.addTestingHooks();
await mergedServer.run();
return;
}
@ -118,17 +119,18 @@ export async function main() {
log.info("== staticServer");
const staticPort = getPort("STATIC_PORT", 9001);
process.env.APP_STATIC_URL = `http://localhost:${staticPort}`;
await mergedServerMain(staticPort, ["static"]);
await MergedServer.create(staticPort, ["static"]).then((s) => s.run());
// Bring up a home server
log.info("==========================================================================");
log.info("== homeServer");
const home = await mergedServerMain(homeServerPort, ["home"]);
const homeServer = await MergedServer.create(homeServerPort, ["home"]);
await homeServer.run();
// If a distinct webServerPort is specified, we listen also on that port, though serving
// exactly the same content. This is handy for testing CORS issues.
if (webServerPort !== 0 && webServerPort !== homeServerPort) {
await home.startCopy('webServer', webServerPort);
await homeServer.flexServer.startCopy('webServer', webServerPort);
}
// Bring up the docWorker(s)
@ -147,10 +149,10 @@ export async function main() {
}
const workers = new Array<FlexServer>();
for (const port of ports) {
workers.push(await mergedServerMain(port, ["docs"]));
workers.push((await MergedServer.create(port, ["docs"])).flexServer);
}
await home.addTestingHooks(workers);
await homeServer.flexServer.addTestingHooks(workers);
}

2
app/server/lib/ActiveDocImport.ts
View File

@ -382,7 +382,7 @@ export class ActiveDocImport {
* @param {String} tmpPath: The path from of the original file.
* @param {FileImportOptions} importOptions: File import options.
* @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
* or guessed by the plugin, and `tables`, which is which is a list of objects with information about
* or guessed by the plugin, and `tables`, which is a list of objects with information about
* tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
*/
private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string,

5
app/server/lib/DocStorageManager.ts
View File

@ -10,7 +10,6 @@ import {DocumentUsage} from 'app/common/DocUsage';
import * as gutil from 'app/common/gutil';
import {Comm} from 'app/server/lib/Comm';
import * as docUtils from 'app/server/lib/docUtils';
import {GristServer} from 'app/server/lib/GristServer';
import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
import {IShell} from 'app/server/lib/IShell';
import log from 'app/server/lib/log';
@ -39,10 +38,10 @@ export class DocStorageManager implements IDocStorageManager {
* The file watcher is created if the optComm argument is given.
*/
constructor(private _docsRoot: string, private _samplesRoot?: string,
private _comm?: Comm, gristServer?: GristServer) {
private _comm?: Comm, shell?: IShell) {
// If we have a way to communicate with clients, watch the docsRoot for changes.
this._watcher = null;
this._shell = gristServer?.create.Shell?.() || {
this._shell = shell ?? {
trashItem() { throw new Error('Unable to move document to trash'); },
showItemInFolder() { throw new Error('Unable to show item in folder'); }
};

9
app/server/lib/ExternalStorage.ts
View File

@ -377,6 +377,15 @@ export interface ExternalStorageSettings {
extraPrefix?: string;
}
/**
* Function returning the core ExternalStorage implementation,
* which may then be wrapped in additional layer(s) of ExternalStorage.
* See ICreate.ExternalStorage.
* Uses S3 by default in hosted Grist.
*/
export type ExternalStorageCreator =
(purpose: ExternalStorageSettings["purpose"], extraPrefix: string) => ExternalStorage | undefined;
/**
* The storage mapping we use for our SaaS. A reasonable default, but relies
* on appropriate lifecycle rules being set up in the bucket.

31
app/server/lib/FlexServer.ts
View File

@ -1,7 +1,6 @@
import {ApiError} from 'app/common/ApiError';
import {ICustomWidget} from 'app/common/CustomWidget';
import {delay} from 'app/common/delay';
import {DocCreationInfo} from 'app/common/DocListAPI';
import {commonUrls, encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
sanitizePathTail} from 'app/common/gristUrls';
@ -38,7 +37,6 @@ import {create} from 'app/server/lib/create';
import {addDiscourseConnectEndpoints} from 'app/server/lib/DiscourseConnect';
import {addDocApiRoutes} from 'app/server/lib/DocApi';
import {DocManager} from 'app/server/lib/DocManager';
import {DocStorageManager} from 'app/server/lib/DocStorageManager';
import {DocWorker} from 'app/server/lib/DocWorker';
import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
@ -47,13 +45,11 @@ import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth";
import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer,
RequestWithGrist} from 'app/server/lib/GristServer';
import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions';
import {HostedStorageManager} from 'app/server/lib/HostedStorageManager';
import {IBilling} from 'app/server/lib/IBilling';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
import {InstallAdmin} from 'app/server/lib/InstallAdmin';
import log from 'app/server/lib/log';
import {getLoginSystem} from 'app/server/lib/logins';
import {IPermitStore} from 'app/server/lib/Permit';
import {getAppPathTo, getAppRoot, getInstanceRoot, getUnpackedAppRoot} from 'app/server/lib/places';
import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint';
@ -185,7 +181,7 @@ export class FlexServer implements GristServer {
private _getSignUpRedirectUrl: (req: express.Request, target: URL) => Promise<string>;
private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise<string>;
private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
private _getLoginSystem?: () => Promise<GristLoginSystem>;
private _getLoginSystem: () => Promise<GristLoginSystem>;
// Set once ready() is called
private _isReady: boolean = false;
private _updateManager: UpdateManager;
@ -193,6 +189,7 @@ export class FlexServer implements GristServer {
constructor(public port: number, public name: string = 'flexServer',
public readonly options: FlexServerOptions = {}) {
this._getLoginSystem = create.getLoginSystem;
this.settings = options.settings;
this.app = express();
this.app.set('port', port);
@ -250,7 +247,6 @@ export class FlexServer implements GristServer {
recentItems: [],
};
this.electronServerMethods = {
async importDoc() { throw new Error('not implemented'); },
onDocOpen(cb) {
// currently only a stub.
cb('');
@ -272,11 +268,6 @@ export class FlexServer implements GristServer {
});
}
// Allow overridding the login system.
public setLoginSystem(loginSystem: () => Promise<GristLoginSystem>) {
this._getLoginSystem = loginSystem;
}
public getHost(): string {
return `${this.host}:${this.getOwnPort()}`;
}
@ -405,6 +396,11 @@ export class FlexServer implements GristServer {
return this._auditLogger;
}
public getDocManager(): DocManager {
if (!this._docManager) { throw new Error('no document manager available'); }
return this._docManager;
}
public getTelemetry(): ITelemetry {
if (!this._telemetry) { throw new Error('no telemetry available'); }
return this._telemetry;
@ -1341,12 +1337,15 @@ export class FlexServer implements GristServer {
const workers = this._docWorkerMap;
const docWorkerId = await this._addSelfAsWorker(workers);
const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableExternalStorage, workers,
this._dbManager, this.create);
const storageManager = await this.create.createHostedDocStorageManager(
this.docsRoot, docWorkerId, this._disableExternalStorage, workers, this._dbManager, this.create.ExternalStorage
);
this._storageManager = storageManager;
} else {
const samples = getAppPathTo(this.appRoot, 'public_samples');
const storageManager = new DocStorageManager(this.docsRoot, samples, this._comm, this);
const storageManager = await this.create.createLocalDocStorageManager(
this.docsRoot, samples, this._comm, this.create.Shell?.()
);
this._storageManager = storageManager;
}
@ -2012,8 +2011,7 @@ export class FlexServer implements GristServer {
public resolveLoginSystem() {
return isTestLoginAllowed() ?
getTestLoginSystem() :
(this._getLoginSystem?.() || getLoginSystem());
getTestLoginSystem() : this._getLoginSystem();
}
public addUpdatesCheck() {
@ -2609,7 +2607,6 @@ function noCaching(req: express.Request, res: express.Response, next: express.Ne
// Methods that Electron app relies on.
export interface ElectronServerMethods {
importDoc(filepath: string): Promise<DocCreationInfo>;
onDocOpen(cb: (filePath: string) => void): void;
getUserConfig(): Promise<any>;
updateUserConfig(obj: any): Promise<void>;

18
app/server/lib/HostedStorageManager.ts
View File

@ -12,9 +12,14 @@ import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import {checksumFile} from 'app/server/lib/checksumFile';
import {DocSnapshotInventory, DocSnapshotPruner} from 'app/server/lib/DocSnapshots';
import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {ChecksummedExternalStorage, DELETED_TOKEN, ExternalStorage, Unchanged} from 'app/server/lib/ExternalStorage';
import {
ChecksummedExternalStorage,
DELETED_TOKEN,
ExternalStorage,
ExternalStorageCreator, ExternalStorageSettings,
Unchanged
} from 'app/server/lib/ExternalStorage';
import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager';
import {ICreate} from 'app/server/lib/ICreate';
import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
import {LogMethods} from "app/server/lib/LogMethods";
import {fromCallback} from 'app/server/lib/serverUtils';
@ -51,11 +56,6 @@ export interface HostedStorageOptions {
secondsBeforePush: number;
secondsBeforeFirstRetry: number;
pushDocUpdateTimes: boolean;
// A function returning the core ExternalStorage implementation,
// which may then be wrapped in additional layer(s) of ExternalStorage.
// See ICreate.ExternalStorage.
// Uses S3 by default in hosted Grist.
externalStorageCreator?: (purpose: 'doc'|'meta') => ExternalStorage;
}
const defaultOptions: HostedStorageOptions = {
@ -134,10 +134,10 @@ export class HostedStorageManager implements IDocStorageManager {
private _disableS3: boolean,
private _docWorkerMap: IDocWorkerMap,
dbManager: HomeDBManager,
create: ICreate,
createExternalStorage: ExternalStorageCreator,
options: HostedStorageOptions = defaultOptions
) {
const creator = options.externalStorageCreator || ((purpose) => create.ExternalStorage(purpose, ''));
const creator = ((purpose: ExternalStorageSettings['purpose']) => createExternalStorage(purpose, ''));
// We store documents either in a test store, or in an s3 store
// at s3://<s3Bucket>/<s3Prefix><docId>.grist
const externalStoreDoc = this._disableS3 ? undefined : creator('doc');

63
app/server/lib/ICreate.ts
View File

@ -1,10 +1,11 @@
import {GristDeploymentType} from 'app/common/gristUrls';
import {getCoreLoginSystem} from 'app/server/lib/coreLogins';
import {getThemeBackgroundSnippet} from 'app/common/Themes';
import {Document} from 'app/gen-server/entity/Document';
import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import {IAuditLogger} from 'app/server/lib/AuditLogger';
import {ExternalStorage} from 'app/server/lib/ExternalStorage';
import {createDummyAuditLogger, createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
import {ExternalStorage, ExternalStorageCreator} from 'app/server/lib/ExternalStorage';
import {createDummyAuditLogger, createDummyTelemetry, GristLoginSystem, GristServer} from 'app/server/lib/GristServer';
import {IBilling} from 'app/server/lib/IBilling';
import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
import {InstallAdmin, SimpleInstallAdmin} from 'app/server/lib/InstallAdmin';
@ -13,6 +14,11 @@ import {IShell} from 'app/server/lib/IShell';
import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
import {SqliteVariant} from 'app/server/lib/SqliteCommon';
import {ITelemetry} from 'app/server/lib/Telemetry';
import {IDocStorageManager} from './IDocStorageManager';
import { Comm } from "./Comm";
import { IDocWorkerMap } from "./DocWorkerMap";
import { HostedStorageManager, HostedStorageOptions } from "./HostedStorageManager";
import { DocStorageManager } from "./DocStorageManager";
// In the past, the session secret was used as an additional
// protection passed on to expressjs-session for security when
@ -37,7 +43,30 @@ import {ITelemetry} from 'app/server/lib/Telemetry';
export const DEFAULT_SESSION_SECRET =
'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
export type LocalDocStorageManagerCreator =
(docsRoot: string, samplesRoot?: string, comm?: Comm, shell?: IShell) => Promise<IDocStorageManager>;
export type HostedDocStorageManagerCreator = (
docsRoot: string,
docWorkerId: string,
disableS3: boolean,
docWorkerMap: IDocWorkerMap,
dbManager: HomeDBManager,
createExternalStorage: ExternalStorageCreator,
options?: HostedStorageOptions
) => Promise<IDocStorageManager>;
export interface ICreate {
// Create a space to store files externally, for storing either:
// - documents. This store should be versioned, and can be eventually consistent.
// - meta. This store need not be versioned, and can be eventually consistent.
// For test purposes an extra prefix may be supplied. Stores with different prefixes
// should not interfere with each other.
ExternalStorage: ExternalStorageCreator;
// Creates a IDocStorageManager for storing documents on the local machine.
createLocalDocStorageManager: LocalDocStorageManagerCreator;
// Creates a IDocStorageManager for storing documents on an external storage (e.g S3)
createHostedDocStorageManager: HostedDocStorageManagerCreator;
Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
@ -45,13 +74,6 @@ export interface ICreate {
Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
Shell?(): IShell; // relevant to electron version of Grist only.
// Create a space to store files externally, for storing either:
// - documents. This store should be versioned, and can be eventually consistent.
// - meta. This store need not be versioned, and can be eventually consistent.
// For test purposes an extra prefix may be supplied. Stores with different prefixes
// should not interfere with each other.
ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage | undefined;
NSandbox(options: ISandboxCreationOptions): ISandbox;
// Create the logic to determine which users are authorized to manage this Grist installation.
@ -69,6 +91,8 @@ export interface ICreate {
getStorageOptions?(name: string): ICreateStorageOptions|undefined;
getSqliteVariant?(): SqliteVariant;
getSandboxVariants?(): Record<string, SpawnFn>;
getLoginSystem(): Promise<GristLoginSystem>;
}
export interface ICreateActiveDocOptions {
@ -126,6 +150,9 @@ export function makeSimpleCreator(opts: {
getSqliteVariant?: () => SqliteVariant,
getSandboxVariants?: () => Record<string, SpawnFn>,
createInstallAdmin?: (dbManager: HomeDBManager) => Promise<InstallAdmin>,
getLoginSystem?: () => Promise<GristLoginSystem>,
createHostedDocStorageManager?: HostedDocStorageManagerCreator,
createLocalDocStorageManager?: LocalDocStorageManagerCreator,
}): ICreate {
const {deploymentType, sessionSecret, storage, notifier, billing, auditLogger, telemetry} = opts;
return {
@ -199,5 +226,23 @@ export function makeSimpleCreator(opts: {
getSqliteVariant: opts.getSqliteVariant,
getSandboxVariants: opts.getSandboxVariants,
createInstallAdmin: opts.createInstallAdmin || (async (dbManager) => new SimpleInstallAdmin(dbManager)),
getLoginSystem: opts.getLoginSystem || getCoreLoginSystem,
createLocalDocStorageManager: opts.createLocalDocStorageManager ?? createDefaultLocalStorageManager,
createHostedDocStorageManager: opts.createHostedDocStorageManager ?? createDefaultHostedStorageManager,
};
}
const createDefaultHostedStorageManager: HostedDocStorageManagerCreator = async (
docsRoot,
docWorkerId,
disableS3,
docWorkerMap,
dbManager,
createExternalStorage, options
) =>
new HostedStorageManager(docsRoot, docWorkerId, disableS3, docWorkerMap, dbManager, createExternalStorage, options);
const createDefaultLocalStorageManager: LocalDocStorageManagerCreator = async (
docsRoot, samplesRoot, comm, shell
) => new DocStorageManager(docsRoot, samplesRoot, comm, shell);

230
app/server/mergedServerMain.ts
View File

@ -1,230 +0,0 @@
/**
*
* A version of hosted grist that recombines a home server,
* a doc worker, and a static server on a single port.
*
*/
import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
import {GristLoginSystem} from 'app/server/lib/GristServer';
import log from 'app/server/lib/log';
import {getGlobalConfig} from "app/server/lib/globalConfig";
// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
// environment variable.
export type ServerType = "home" | "docs" | "static" | "app";
const allServerTypes: ServerType[] = ["home", "docs", "static", "app"];
// Parse a comma-separate list of server types into an array, with validation.
export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
// Split and filter out empty strings (including the one we get when splitting "").
const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part));
// Check that parts is non-empty and only contains valid options.
if (!types.length) {
throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`);
}
for (const t of types) {
if (!allServerTypes.includes(t as ServerType)) {
throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`);
}
}
return types as ServerType[];
}
function checkUserContentPort(): number | null {
// Check whether a port is explicitly set for user content.
if (process.env.GRIST_UNTRUSTED_PORT) {
return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
}
// Checks whether to serve user content on same domain but on different port
if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
const homeUrl = new URL(process.env.APP_HOME_URL);
const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
// If the hostname of both home and plugin url are the same,
// but the ports are different
if (homeUrl.hostname === pluginUrl.hostname &&
homeUrl.port !== pluginUrl.port) {
const port = parseInt(pluginUrl.port || '80', 10);
return port;
}
}
return null;
}
interface ServerOptions extends FlexServerOptions {
logToConsole?: boolean; // If set, messages logged to console (default: false)
// (but if options are not given at all in call to main,
// logToConsole is set to true)
externalStorage?: boolean; // If set, documents saved to external storage such as s3 (default is to check environment
// variables, which get set in various ways in dev/test entry points)
loginSystem?: () => Promise<GristLoginSystem>;
}
/**
* Start a server on the given port, including the functionality specified in serverTypes.
*/
export async function main(port: number, serverTypes: ServerType[],
options: ServerOptions = {}) {
const includeHome = serverTypes.includes("home");
const includeDocs = serverTypes.includes("docs");
const includeStatic = serverTypes.includes("static");
const includeApp = serverTypes.includes("app");
options.settings ??= getGlobalConfig();
const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
// We need to know early on whether we will be serving plugins or not.
if (includeHome) {
const userPort = checkUserContentPort();
server.setServesPlugins(userPort !== undefined);
} else {
server.setServesPlugins(false);
}
if (options.loginSystem) {
server.setLoginSystem(options.loginSystem);
}
server.addCleanup();
server.setDirectory();
if (process.env.GRIST_TEST_ROUTER) {
// Add a mock api for adding/removing doc workers from load balancer.
server.testAddRouter();
}
if (options.logToConsole !== false) { server.addLogging(); }
if (options.externalStorage === false) { server.disableExternalStorage(); }
await server.addLoginMiddleware();
if (includeDocs) {
// It is important that /dw and /v prefixes are accepted (if present) by health check
// in this case, since they are included in the url registered for the doc worker.
server.stripDocWorkerIdPathPrefixIfPresent();
server.addTagChecker();
}
server.addHealthCheck();
if (includeHome || includeApp) {
server.addBootPage();
}
server.denyRequestsIfNotReady();
if (includeHome || includeStatic || includeApp) {
server.setDirectory();
}
if (includeHome || includeStatic) {
server.addStaticAndBowerDirectories();
}
await server.initHomeDBManager();
server.addHosts();
server.addDocWorkerMap();
if (includeHome || includeStatic) {
await server.addAssetsForPlugins();
}
if (includeHome) {
server.addEarlyWebhooks();
}
if (includeHome || includeDocs || includeApp) {
server.addSessions();
}
server.addAccessMiddleware();
server.addApiMiddleware();
await server.addBillingMiddleware();
try {
await server.start();
if (includeHome) {
server.addUsage();
if (!includeDocs) {
server.addDocApiForwarder();
}
server.addJsonSupport();
server.addUpdatesCheck();
await server.addLandingPages();
// todo: add support for home api to standalone app
server.addHomeApi();
server.addBillingApi();
server.addNotifier();
server.addAuditLogger();
await server.addTelemetry();
await server.addHousekeeper();
await server.addLoginRoutes();
server.addAccountPage();
server.addBillingPages();
server.addWelcomePaths();
server.addLogEndpoint();
server.addGoogleAuthEndpoint();
server.addInstallEndpoints();
server.addConfigEndpoints();
}
if (includeDocs) {
server.addJsonSupport();
server.addAuditLogger();
await server.addTelemetry();
await server.addDoc();
}
if (includeHome) {
server.addClientSecrets();
}
server.finalizeEndpoints();
await server.finalizePlugins(includeHome ? checkUserContentPort() : null);
server.checkOptionCombinations();
server.summary();
server.ready();
// Some tests have their timing perturbed by having this earlier
// TODO: update those tests.
if (includeDocs) {
await server.checkSandbox();
}
return server;
} catch(e) {
await server.close();
throw e;
}
}
export async function startMain() {
try {
const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
// No defaults for a port, since this server can serve very different purposes.
if (!process.env.GRIST_PORT) {
throw new Error("GRIST_PORT must be specified");
}
const port = parseInt(process.env.GRIST_PORT, 10);
const server = await main(port, serverTypes);
const opt = process.argv[2];
if (opt === '--testingHooks') {
await server.addTestingHooks();
}
return server;
} catch (e) {
log.error('mergedServer failed to start', e);
process.exit(1);
}
}
if (require.main === module) {
startMain().catch((e) => log.error('mergedServer failed to start', e));
}

4
app/server/utils/pruneActionHistory.ts
View File

@ -1,9 +1,9 @@
import * as gutil from 'app/common/gutil';
import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl';
import {DocStorage} from 'app/server/lib/DocStorage';
import {DocStorageManager} from 'app/server/lib/DocStorageManager';
import * as docUtils from 'app/server/lib/docUtils';
import log from 'app/server/lib/log';
import {create} from "app/server/lib/create";
/**
* A utility script for cleaning up the action log.
@ -18,7 +18,7 @@ export async function pruneActionHistory(docPath: string, keepN: number) {
throw new Error('Invalid document: Document should be a valid .grist file');
}
const storageManager = new DocStorageManager(".", ".");
const storageManager = await create.createLocalDocStorageManager(".", ".");
const docStorage = new DocStorage(storageManager, docPath);
const backupPath = gutil.removeSuffix(docPath, '.grist') + "-backup.grist";

9
sandbox/watch.sh
View File

@ -2,6 +2,13 @@
set -x
NO_NODEMON=false
for arg in $@; do
if [[ $arg == "--no-nodemon" ]]; then
NO_NODEMON=true
fi
done
PROJECT=""
if [[ -e ext/app ]]; then
PROJECT="tsconfig-ext.json"
@ -19,6 +26,6 @@ tsc --build -w --preserveWatchOutput $PROJECT &
css_files="app/client/**/*.css"
chokidar "${css_files}" -c "bash -O globstar -c 'cat ${css_files} > static/bundle.css'" &
webpack --config $WEBPACK_CONFIG --mode development --watch &
NODE_PATH=_build:_build/stubs:_build/ext nodemon ${NODE_INSPECT} --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js &
! $NO_NODEMON && NODE_PATH=_build:_build/stubs:_build/ext nodemon ${NODE_INSPECT} --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js &
wait

2
stubs/app/client/ui/HomeImports.ts Normal file
View File

@ -0,0 +1,2 @@
import * as coreHomeImports from "app/client/ui/CoreHomeImports";
export const homeImports = coreHomeImports;

2
stubs/app/client/ui/NewDocMethods.ts Normal file
View File

@ -0,0 +1,2 @@
import * as coreNewDocMethods from "app/client/ui/CoreNewDocMethods";
export const newDocMethods = coreNewDocMethods;

6
stubs/app/server/lib/logins.ts
View File

@ -1,6 +0,0 @@
import { getCoreLoginSystem } from "app/server/lib/coreLogins";
import { GristLoginSystem } from "app/server/lib/GristServer";
export async function getLoginSystem(): Promise<GristLoginSystem> {
return getCoreLoginSystem();
}

16
stubs/app/server/server.ts
View File

@ -34,7 +34,7 @@ setDefaultEnv('GRIST_UI_FEATURES',
'helpCenter,billing,templates,multiSite,multiAccounts,sendToDrive,createSite,supportGrist');
setDefaultEnv('GRIST_WIDGET_LIST_URL', commonUrls.gristLabsWidgetRepository);
import {updateDb} from 'app/server/lib/dbUtils';
import {main as mergedServerMain, parseServerTypes} from 'app/server/mergedServerMain';
import {MergedServer, parseServerTypes} from 'app/server/MergedServer';
import * as fse from 'fs-extra';
import {runPrometheusExporter} from './prometheus-exporter';
@ -124,20 +124,20 @@ export async function main() {
}
// Launch single-port, self-contained version of Grist.
const server = await mergedServerMain(G.port, serverTypes);
const mergedServer = await MergedServer.create(G.port, serverTypes);
await mergedServer.run();
if (process.env.GRIST_TESTING_SOCKET) {
await server.addTestingHooks();
await mergedServer.flexServer.addTestingHooks();
}
if (process.env.GRIST_SERVE_PLUGINS_PORT) {
await server.startCopy('pluginServer', parseInt(process.env.GRIST_SERVE_PLUGINS_PORT, 10));
await mergedServer.flexServer.startCopy('pluginServer', parseInt(process.env.GRIST_SERVE_PLUGINS_PORT, 10));
}
await fixSiteProducts({
deploymentType: server.getDeploymentType(),
db: server.getHomeDBManager()
deploymentType: mergedServer.flexServer.getDeploymentType(),
db: mergedServer.flexServer.getHomeDBManager()
});
return server;
return mergedServer.flexServer;
}
if (require.main === module) {

11
test/gen-server/AuthCaching.ts
View File

@ -2,7 +2,7 @@ import {delay} from 'app/common/delay';
import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import {FlexServer} from 'app/server/lib/FlexServer';
import log from 'app/server/lib/log';
import {main as mergedServerMain} from 'app/server/mergedServerMain';
import {MergedServer} from 'app/server/MergedServer';
import axios from 'axios';
import {assert} from 'chai';
import * as fse from 'fs-extra';
@ -50,12 +50,17 @@ describe('AuthCaching', function() {
setUpDB();
await createInitialDb();
process.env.GRIST_DATA_DIR = testDocDir;
homeServer = await mergedServerMain(0, ['home'],
const homeMS = await MergedServer.create(0, ['home'],
{logToConsole: false, externalStorage: false});
await homeMS.run();
homeServer = homeMS.flexServer;
homeUrl = homeServer.getOwnUrl();
process.env.APP_HOME_URL = homeUrl;
docsServer = await mergedServerMain(0, ['docs'],
const docsMS = await MergedServer.create(0, ['docs'],
{logToConsole: false, externalStorage: false});
await docsMS.run();
docsServer = docsMS.flexServer;
// Helpers for getting cookie-based logins.
session = new TestSession(homeServer);

11
test/gen-server/apiUtils.ts
View File

@ -14,7 +14,7 @@ import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import * as docUtils from 'app/server/lib/docUtils';
import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
import {main as mergedServerMain, ServerType} from 'app/server/mergedServerMain';
import {MergedServer, ServerType} from 'app/server/MergedServer';
import axios from 'axios';
import FormData from 'form-data';
import fetch from 'node-fetch';
@ -37,9 +37,10 @@ export class TestServer {
public async start(servers: ServerType[] = ["home"],
options: FlexServerOptions = {}): Promise<string> {
await createInitialDb();
this.server = await mergedServerMain(0, servers, {logToConsole: isAffirmative(process.env.DEBUG),
externalStorage: false,
...options});
const mergedServer = await MergedServer.create(0, servers, {logToConsole: isAffirmative(process.env.DEBUG),
externalStorage: false, ...options});
await mergedServer.run();
this.server = mergedServer.flexServer;
this.serverUrl = this.server.getOwnUrl();
this.dbManager = this.server.getHomeDBManager();
this.defaultSession = new TestSession(this.server);
@ -263,7 +264,7 @@ export class TestSession {
if (clearCache) { this.home.getSessions().clearCacheIfNeeded(); }
this.headers.Cookie = cookie;
return {
validateStatus: (status: number) => true,
validateStatus: (_status: number) => true,
headers: {
'Cookie': cookie,
'X-Requested-With': 'XMLHttpRequest',

22
test/gen-server/lib/DocWorkerMap.ts
View File

@ -2,7 +2,7 @@ import {DocWorkerMap, getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {FlexServer} from 'app/server/lib/FlexServer';
import {Permit} from 'app/server/lib/Permit';
import {main as mergedServerMain} from 'app/server/mergedServerMain';
import {MergedServer} from "app/server/MergedServer";
import {delay, promisifyAll} from 'bluebird';
import {assert, expect} from 'chai';
import {countBy, values} from 'lodash';
@ -387,24 +387,34 @@ describe('DocWorkerMap', function() {
process.env.REDIS_URL = process.env.TEST_REDIS_URL;
// Make home server.
const home = await mergedServerMain(0, ['home'], opts);
const homeMergedServer = await MergedServer.create(0, ['home'], opts);
const home = homeMergedServer.flexServer;
await homeMergedServer.run();
// Make a worker, not associated with any group.
process.env.GRIST_DOC_WORKER_ID = 'worker1';
const docs1 = await mergedServerMain(0, ['docs'], opts);
const docs1MergedServer = await MergedServer.create(0, ['docs'], opts);
const docs1 = docs1MergedServer.flexServer;
await docs1MergedServer.run();
// Make a worker in "special" group.
process.env.GRIST_DOC_WORKER_ID = 'worker2';
process.env.GRIST_WORKER_GROUP = 'special';
const docs2 = await mergedServerMain(0, ['docs'], opts);
const docs2MergedServer = await MergedServer.create(0, ['docs'], opts);
const docs2 = docs2MergedServer.flexServer;
await docs2MergedServer.run();
// Make two worker in "other" group.
process.env.GRIST_DOC_WORKER_ID = 'worker3';
process.env.GRIST_WORKER_GROUP = 'other';
const docs3 = await mergedServerMain(0, ['docs'], opts);
const docs3MergedServer = await MergedServer.create(0, ['docs'], opts);
const docs3 = docs3MergedServer.flexServer;
await docs3MergedServer.run();
process.env.GRIST_DOC_WORKER_ID = 'worker4';
process.env.GRIST_WORKER_GROUP = 'other';
const docs4 = await mergedServerMain(0, ['docs'], opts);
const docs4MergedServer = await MergedServer.create(0, ['docs'], opts);
const docs4 = docs4MergedServer.flexServer;
await docs4MergedServer.run();
servers = {home, docs1, docs2, docs3, docs4};
workers = getDocWorkerMap();

7
test/gen-server/lib/mergedOrgs.ts
View File

@ -1,7 +1,7 @@
import {Workspace} from 'app/common/UserAPI';
import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import {FlexServer} from 'app/server/lib/FlexServer';
import {main as mergedServerMain} from 'app/server/mergedServerMain';
import {MergedServer} from "app/server/MergedServer";
import axios from 'axios';
import {assert} from 'chai';
import {createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed';
@ -9,6 +9,7 @@ import {configForUser, createUser, setPlan} from 'test/gen-server/testUtils';
import * as testUtils from 'test/server/testUtils';
describe('mergedOrgs', function() {
let mergedServer: MergedServer;
let home: FlexServer;
let dbManager: HomeDBManager;
let homeUrl: string;
@ -20,8 +21,10 @@ describe('mergedOrgs', function() {
before(async function() {
setUpDB(this);
await createInitialDb();
home = await mergedServerMain(0, ["home", "docs"],
mergedServer = await MergedServer.create(0, ["home", "docs"],
{logToConsole: false, externalStorage: false});
home = mergedServer.flexServer;
await mergedServer.run();
dbManager = home.getHomeDBManager();
homeUrl = home.getOwnUrl();
});

4
test/server/docTools.ts
View File

@ -4,7 +4,6 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {DummyAuthorizer} from 'app/server/lib/Authorizer';
import {DocManager} from 'app/server/lib/DocManager';
import {DocSession, makeExceptionalDocSession} from 'app/server/lib/DocSession';
import {DocStorageManager} from 'app/server/lib/DocStorageManager';
import {createDummyGristServer, GristServer} from 'app/server/lib/GristServer';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import {getAppRoot} from 'app/server/lib/places';
@ -17,6 +16,7 @@ import * as fse from 'fs-extra';
import {tmpdir} from 'os';
import * as path from 'path';
import * as tmp from 'tmp';
import {create} from "app/server/lib/create";
tmp.setGracefulCleanup();
@ -138,7 +138,7 @@ export async function createDocManager(
server?: GristServer} = {}): Promise<DocManager> {
// Set Grist home to a temporary directory, and wipe it out on exit.
const tmpDir = options.tmpDir || await createTmpDir();
const docStorageManager = options.storageManager || new DocStorageManager(tmpDir);
const docStorageManager = options.storageManager || await create.createLocalDocStorageManager(tmpDir);
const pluginManager = options.pluginManager || await getGlobalPluginManager();
const store = getDocWorkerMap();
const internalPermitStore = store.getPermitStore('1');

17
test/server/lib/HostedStorageManager.ts
View File

@ -7,7 +7,12 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {create} from 'app/server/lib/create';
import {DocManager} from 'app/server/lib/DocManager';
import {makeExceptionalDocSession} from 'app/server/lib/DocSession';
import {DELETED_TOKEN, ExternalStorage, wrapWithKeyMappedStorage} from 'app/server/lib/ExternalStorage';
import {
DELETED_TOKEN,
ExternalStorage, ExternalStorageCreator,
ExternalStorageSettings,
wrapWithKeyMappedStorage
} from 'app/server/lib/ExternalStorage';
import {createDummyGristServer} from 'app/server/lib/GristServer';
import {
BackupEvent,
@ -270,7 +275,7 @@ class TestStore {
private _localDirectory: string,
private _workerId: string,
private _workers: DocWorkerMap,
private _externalStorageCreate: (purpose: 'doc'|'meta', extraPrefix: string) => ExternalStorage|undefined) {
private _externalStorageCreate: ExternalStorageCreator) {
}
public async run<T>(fn: () => Promise<T>): Promise<T> {
@ -296,18 +301,20 @@ class TestStore {
secondsBeforeFirstRetry: 3, // rumors online suggest delays of 10-11 secs
// are not super-unusual.
pushDocUpdateTimes: false,
externalStorageCreator: (purpose) => {
};
const externalStorageCreator = (purpose: ExternalStorageSettings["purpose"]) => {
const result = this._externalStorageCreate(purpose, this._extraPrefix);
if (!result) { throw new Error('no storage'); }
return result;
}
};
const storageManager = new HostedStorageManager(this._localDirectory,
this._workerId,
false,
this._workers,
dbManager,
create,
externalStorageCreator,
options);
this.storageManager = storageManager;
this.docManager = new DocManager(storageManager, await getGlobalPluginManager(),

2
test/server/lib/helpers/TestServer.ts
View File

@ -91,7 +91,7 @@ export class TestServer {
...this._defaultEnv,
...customEnv
};
const main = await testUtils.getBuildFile('app/server/mergedServerMain.js');
const main = await testUtils.getBuildFile('app/server/MergedServer.js');
this._server = spawn('node', [main, '--testingHooks'], {
env,
stdio: ['inherit', serverLog, serverLog]