Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

Make changes required for Desktop FS updates (#1099)

Browse Source 
Make a set of changes required for Desktop FS improvements, see
https://github.com/gristlabs/grist-desktop/pull/42

---------

Co-authored-by: Spoffy <contact@spoffy.net>
Co-authored-by: Spoffy <4805393+Spoffy@users.noreply.github.com>
This commit is contained in:
Leslie H
2024-09-17 01:01:58 +00:00
committed by GitHub
parent 938bb0666e
commit 02cfcee84d
29 changed files with 552 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/server/lib/ActiveDocImport.ts
View File

@@ -382,7 +382,7 @@ export class ActiveDocImport {
* @param {String} tmpPath: The path from of the original file.
* @param {FileImportOptions} importOptions: File import options.
* @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
* or guessed by the plugin, and `tables`, which is which is a list of objects with information about
* or guessed by the plugin, and `tables`, which is a list of objects with information about
* tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
*/
private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string,

5
app/server/lib/DocStorageManager.ts
View File

@@ -10,7 +10,6 @@ import {DocumentUsage} from 'app/common/DocUsage';
import * as gutil from 'app/common/gutil';
import {Comm} from 'app/server/lib/Comm';
import * as docUtils from 'app/server/lib/docUtils';
import {GristServer} from 'app/server/lib/GristServer';
import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
import {IShell} from 'app/server/lib/IShell';
import log from 'app/server/lib/log';
@@ -39,10 +38,10 @@ export class DocStorageManager implements IDocStorageManager {
* The file watcher is created if the optComm argument is given.
*/
constructor(private _docsRoot: string, private _samplesRoot?: string,
private _comm?: Comm, gristServer?: GristServer) {
private _comm?: Comm, shell?: IShell) {
// If we have a way to communicate with clients, watch the docsRoot for changes.
this._watcher = null;
this._shell = gristServer?.create.Shell?.() || {
this._shell = shell ?? {
trashItem() { throw new Error('Unable to move document to trash'); },
showItemInFolder() { throw new Error('Unable to show item in folder'); }
};

9
app/server/lib/ExternalStorage.ts
View File

@@ -377,6 +377,15 @@ export interface ExternalStorageSettings {
extraPrefix?: string;
}
/**
* Function returning the core ExternalStorage implementation,
* which may then be wrapped in additional layer(s) of ExternalStorage.
* See ICreate.ExternalStorage.
* Uses S3 by default in hosted Grist.
*/
export type ExternalStorageCreator =
(purpose: ExternalStorageSettings["purpose"], extraPrefix: string) => ExternalStorage | undefined;
/**
* The storage mapping we use for our SaaS. A reasonable default, but relies
* on appropriate lifecycle rules being set up in the bucket.

31
app/server/lib/FlexServer.ts
View File

@@ -1,7 +1,6 @@
import {ApiError} from 'app/common/ApiError';
import {ICustomWidget} from 'app/common/CustomWidget';
import {delay} from 'app/common/delay';
import {DocCreationInfo} from 'app/common/DocListAPI';
import {commonUrls, encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
sanitizePathTail} from 'app/common/gristUrls';
@@ -38,7 +37,6 @@ import {create} from 'app/server/lib/create';
import {addDiscourseConnectEndpoints} from 'app/server/lib/DiscourseConnect';
import {addDocApiRoutes} from 'app/server/lib/DocApi';
import {DocManager} from 'app/server/lib/DocManager';
import {DocStorageManager} from 'app/server/lib/DocStorageManager';
import {DocWorker} from 'app/server/lib/DocWorker';
import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
@@ -47,13 +45,11 @@ import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth";
import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer,
RequestWithGrist} from 'app/server/lib/GristServer';
import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions';
import {HostedStorageManager} from 'app/server/lib/HostedStorageManager';
import {IBilling} from 'app/server/lib/IBilling';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
import {InstallAdmin} from 'app/server/lib/InstallAdmin';
import log from 'app/server/lib/log';
import {getLoginSystem} from 'app/server/lib/logins';
import {IPermitStore} from 'app/server/lib/Permit';
import {getAppPathTo, getAppRoot, getInstanceRoot, getUnpackedAppRoot} from 'app/server/lib/places';
import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint';
@@ -185,7 +181,7 @@ export class FlexServer implements GristServer {
private _getSignUpRedirectUrl: (req: express.Request, target: URL) => Promise<string>;
private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise<string>;
private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
private _getLoginSystem?: () => Promise<GristLoginSystem>;
private _getLoginSystem: () => Promise<GristLoginSystem>;
// Set once ready() is called
private _isReady: boolean = false;
private _updateManager: UpdateManager;
@@ -193,6 +189,7 @@ export class FlexServer implements GristServer {
constructor(public port: number, public name: string = 'flexServer',
public readonly options: FlexServerOptions = {}) {
this._getLoginSystem = create.getLoginSystem;
this.settings = options.settings;
this.app = express();
this.app.set('port', port);
@@ -250,7 +247,6 @@ export class FlexServer implements GristServer {
recentItems: [],
};
this.electronServerMethods = {
async importDoc() { throw new Error('not implemented'); },
onDocOpen(cb) {
// currently only a stub.
cb('');
@@ -272,11 +268,6 @@ export class FlexServer implements GristServer {
});
}
// Allow overridding the login system.
public setLoginSystem(loginSystem: () => Promise<GristLoginSystem>) {
this._getLoginSystem = loginSystem;
}
public getHost(): string {
return `${this.host}:${this.getOwnPort()}`;
}
@@ -405,6 +396,11 @@ export class FlexServer implements GristServer {
return this._auditLogger;
}
public getDocManager(): DocManager {
if (!this._docManager) { throw new Error('no document manager available'); }
return this._docManager;
}
public getTelemetry(): ITelemetry {
if (!this._telemetry) { throw new Error('no telemetry available'); }
return this._telemetry;
@@ -1341,12 +1337,15 @@ export class FlexServer implements GristServer {
const workers = this._docWorkerMap;
const docWorkerId = await this._addSelfAsWorker(workers);
const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableExternalStorage, workers,
this._dbManager, this.create);
const storageManager = await this.create.createHostedDocStorageManager(
this.docsRoot, docWorkerId, this._disableExternalStorage, workers, this._dbManager, this.create.ExternalStorage
);
this._storageManager = storageManager;
} else {
const samples = getAppPathTo(this.appRoot, 'public_samples');
const storageManager = new DocStorageManager(this.docsRoot, samples, this._comm, this);
const storageManager = await this.create.createLocalDocStorageManager(
this.docsRoot, samples, this._comm, this.create.Shell?.()
);
this._storageManager = storageManager;
}
@@ -2012,8 +2011,7 @@ export class FlexServer implements GristServer {
public resolveLoginSystem() {
return isTestLoginAllowed() ?
getTestLoginSystem() :
(this._getLoginSystem?.() || getLoginSystem());
getTestLoginSystem() : this._getLoginSystem();
}
public addUpdatesCheck() {
@@ -2609,7 +2607,6 @@ function noCaching(req: express.Request, res: express.Response, next: express.Ne
// Methods that Electron app relies on.
export interface ElectronServerMethods {
importDoc(filepath: string): Promise<DocCreationInfo>;
onDocOpen(cb: (filePath: string) => void): void;
getUserConfig(): Promise<any>;
updateUserConfig(obj: any): Promise<void>;

18
app/server/lib/HostedStorageManager.ts
View File

@@ -12,9 +12,14 @@ import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import {checksumFile} from 'app/server/lib/checksumFile';
import {DocSnapshotInventory, DocSnapshotPruner} from 'app/server/lib/DocSnapshots';
import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {ChecksummedExternalStorage, DELETED_TOKEN, ExternalStorage, Unchanged} from 'app/server/lib/ExternalStorage';
import {
ChecksummedExternalStorage,
DELETED_TOKEN,
ExternalStorage,
ExternalStorageCreator, ExternalStorageSettings,
Unchanged
} from 'app/server/lib/ExternalStorage';
import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager';
import {ICreate} from 'app/server/lib/ICreate';
import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
import {LogMethods} from "app/server/lib/LogMethods";
import {fromCallback} from 'app/server/lib/serverUtils';
@@ -51,11 +56,6 @@ export interface HostedStorageOptions {
secondsBeforePush: number;
secondsBeforeFirstRetry: number;
pushDocUpdateTimes: boolean;
// A function returning the core ExternalStorage implementation,
// which may then be wrapped in additional layer(s) of ExternalStorage.
// See ICreate.ExternalStorage.
// Uses S3 by default in hosted Grist.
externalStorageCreator?: (purpose: 'doc'|'meta') => ExternalStorage;
}
const defaultOptions: HostedStorageOptions = {
@@ -134,10 +134,10 @@ export class HostedStorageManager implements IDocStorageManager {
private _disableS3: boolean,
private _docWorkerMap: IDocWorkerMap,
dbManager: HomeDBManager,
create: ICreate,
createExternalStorage: ExternalStorageCreator,
options: HostedStorageOptions = defaultOptions
) {
const creator = options.externalStorageCreator || ((purpose) => create.ExternalStorage(purpose, ''));
const creator = ((purpose: ExternalStorageSettings['purpose']) => createExternalStorage(purpose, ''));
// We store documents either in a test store, or in an s3 store
// at s3://<s3Bucket>/<s3Prefix><docId>.grist
const externalStoreDoc = this._disableS3 ? undefined : creator('doc');

63
app/server/lib/ICreate.ts
View File

@@ -1,10 +1,11 @@
import {GristDeploymentType} from 'app/common/gristUrls';
import {getCoreLoginSystem} from 'app/server/lib/coreLogins';
import {getThemeBackgroundSnippet} from 'app/common/Themes';
import {Document} from 'app/gen-server/entity/Document';
import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import {IAuditLogger} from 'app/server/lib/AuditLogger';
import {ExternalStorage} from 'app/server/lib/ExternalStorage';
import {createDummyAuditLogger, createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
import {ExternalStorage, ExternalStorageCreator} from 'app/server/lib/ExternalStorage';
import {createDummyAuditLogger, createDummyTelemetry, GristLoginSystem, GristServer} from 'app/server/lib/GristServer';
import {IBilling} from 'app/server/lib/IBilling';
import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
import {InstallAdmin, SimpleInstallAdmin} from 'app/server/lib/InstallAdmin';
@@ -13,6 +14,11 @@ import {IShell} from 'app/server/lib/IShell';
import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
import {SqliteVariant} from 'app/server/lib/SqliteCommon';
import {ITelemetry} from 'app/server/lib/Telemetry';
import {IDocStorageManager} from './IDocStorageManager';
import { Comm } from "./Comm";
import { IDocWorkerMap } from "./DocWorkerMap";
import { HostedStorageManager, HostedStorageOptions } from "./HostedStorageManager";
import { DocStorageManager } from "./DocStorageManager";
// In the past, the session secret was used as an additional
// protection passed on to expressjs-session for security when
@@ -37,7 +43,30 @@ import {ITelemetry} from 'app/server/lib/Telemetry';
export const DEFAULT_SESSION_SECRET =
'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
export type LocalDocStorageManagerCreator =
(docsRoot: string, samplesRoot?: string, comm?: Comm, shell?: IShell) => Promise<IDocStorageManager>;
export type HostedDocStorageManagerCreator = (
docsRoot: string,
docWorkerId: string,
disableS3: boolean,
docWorkerMap: IDocWorkerMap,
dbManager: HomeDBManager,
createExternalStorage: ExternalStorageCreator,
options?: HostedStorageOptions
) => Promise<IDocStorageManager>;
export interface ICreate {
// Create a space to store files externally, for storing either:
// - documents. This store should be versioned, and can be eventually consistent.
// - meta. This store need not be versioned, and can be eventually consistent.
// For test purposes an extra prefix may be supplied. Stores with different prefixes
// should not interfere with each other.
ExternalStorage: ExternalStorageCreator;
// Creates a IDocStorageManager for storing documents on the local machine.
createLocalDocStorageManager: LocalDocStorageManagerCreator;
// Creates a IDocStorageManager for storing documents on an external storage (e.g S3)
createHostedDocStorageManager: HostedDocStorageManagerCreator;
Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
@@ -45,13 +74,6 @@ export interface ICreate {
Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
Shell?(): IShell; // relevant to electron version of Grist only.
// Create a space to store files externally, for storing either:
// - documents. This store should be versioned, and can be eventually consistent.
// - meta. This store need not be versioned, and can be eventually consistent.
// For test purposes an extra prefix may be supplied. Stores with different prefixes
// should not interfere with each other.
ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage | undefined;
NSandbox(options: ISandboxCreationOptions): ISandbox;
// Create the logic to determine which users are authorized to manage this Grist installation.
@@ -69,6 +91,8 @@ export interface ICreate {
getStorageOptions?(name: string): ICreateStorageOptions|undefined;
getSqliteVariant?(): SqliteVariant;
getSandboxVariants?(): Record<string, SpawnFn>;
getLoginSystem(): Promise<GristLoginSystem>;
}
export interface ICreateActiveDocOptions {
@@ -126,6 +150,9 @@ export function makeSimpleCreator(opts: {
getSqliteVariant?: () => SqliteVariant,
getSandboxVariants?: () => Record<string, SpawnFn>,
createInstallAdmin?: (dbManager: HomeDBManager) => Promise<InstallAdmin>,
getLoginSystem?: () => Promise<GristLoginSystem>,
createHostedDocStorageManager?: HostedDocStorageManagerCreator,
createLocalDocStorageManager?: LocalDocStorageManagerCreator,
}): ICreate {
const {deploymentType, sessionSecret, storage, notifier, billing, auditLogger, telemetry} = opts;
return {
@@ -199,5 +226,23 @@ export function makeSimpleCreator(opts: {
getSqliteVariant: opts.getSqliteVariant,
getSandboxVariants: opts.getSandboxVariants,
createInstallAdmin: opts.createInstallAdmin || (async (dbManager) => new SimpleInstallAdmin(dbManager)),
getLoginSystem: opts.getLoginSystem || getCoreLoginSystem,
createLocalDocStorageManager: opts.createLocalDocStorageManager ?? createDefaultLocalStorageManager,
createHostedDocStorageManager: opts.createHostedDocStorageManager ?? createDefaultHostedStorageManager,
};
}
const createDefaultHostedStorageManager: HostedDocStorageManagerCreator = async (
docsRoot,
docWorkerId,
disableS3,
docWorkerMap,
dbManager,
createExternalStorage, options
) =>
new HostedStorageManager(docsRoot, docWorkerId, disableS3, docWorkerMap, dbManager, createExternalStorage, options);
const createDefaultLocalStorageManager: LocalDocStorageManagerCreator = async (
docsRoot, samplesRoot, comm, shell
) => new DocStorageManager(docsRoot, samplesRoot, comm, shell);