(core) make it easier to enable Azure storage without setting GRIST_DOCS_S3_BUCKET

Summary:
Previously, absence of `GRIST_DOCS_S3_BUCKET` was equated with absence
of external storage, but that is no longer true now that Azure is
available. Azure could be used by setting `GRIST_DOCS_S3_BUCKET`
but the alternative `GRIST_AZURE_CONTAINER` flag is friendlier.

Test Plan:
confirmed manually that Azure can be configured and
used now without `GRIST_DOCS_S3_BUCKET`

Reviewers: alexmojaki

Reviewed By: alexmojaki

Subscribers: alexmojaki

Differential Revision: https://phab.getgrist.com/D3448
pull/214/head
Paul Fitzpatrick 2 years ago
parent acddd25cfd
commit 1c6f80f956

@ -0,0 +1,204 @@
import { isAffirmative } from 'app/common/gutil';
/**
* A bundle of settings for the application. May contain
* a value directly, and/or via nested settings. Also
* may have some information about where we looked for
* the value, for reporting as a diagnostic.
*/
export class AppSettings {
private _value?: JSONValue;
private _children?: {[key: string]: AppSettings};
private _info?: AppSettingQueryResult;
public constructor(public readonly name: string) {}
/* access the setting - undefined if not set */
public get(): JSONValue|undefined {
return this._value;
}
/* access the setting as a boolean using isAffirmative - undefined if not set */
public getAsBool(): boolean|undefined {
return (this._value !== undefined) ? isAffirmative(this._value) : undefined;
}
/**
* Try to read the setting from the environment. Even if
* we fail, we record information about how we tried to
* find the setting, so we can report on that.
*/
public read(query: AppSettingQuery) {
this._value = undefined;
this._info = undefined;
let value = undefined;
let found = false;
const envVars = getEnvVarsFromQuery(query);
if (!envVars.length) {
throw new Error('could not find an environment variable to read');
}
let envVar = envVars[0];
for (const synonym of envVars) {
value = process.env[synonym];
if (value !== undefined) {
envVar = synonym;
found = true;
break;
}
}
this._info = {
envVar: found ? envVar : undefined,
found,
query,
};
if (value !== undefined) {
this._value = value;
} else if (query.defaultValue !== undefined) {
this._value = query.defaultValue;
}
return this;
}
/**
* As for read() but type the result as a string.
*/
public readString(query: AppSettingQuery): string|undefined {
this.read(query);
if (this._value === undefined) { return undefined; }
this._value = String(this._value);
return this._value;
}
/**
* As for readString() but fail if nothing was found.
*/
public requireString(query: AppSettingQuery): string {
const result = this.readString(query);
if (result === undefined) {
throw new Error(`missing environment variable: ${query.envVar}`);
}
return result;
}
/* set this setting 'manually' */
public set(value: JSONValue): void {
this._value = value;
this._info = undefined;
}
/* access any nested settings */
public get nested(): {[key: string]: AppSettings} {
return this._children || {};
}
/**
* Add a named nested setting, returning an AppSettings
* object that can be used to access it. This method is
* named "section" to suggest that the nested setting
* will itself contain multiple settings, but doesn't
* require that.
*/
public section(fname: string): AppSettings {
if (!this._children) { this._children = {}; }
let child = this._children[fname];
if (!child) {
this._children[fname] = child = new AppSettings(fname);
}
return child;
}
/**
* Add a named nested setting, returning an AppSettings
* object that can be used to access it. This method is
* named "flag" to suggest that tthe nested setting will
* not iself be nested, but doesn't require that - it is
* currently just an alias for the section() method.
*/
public flag(fname: string): AppSettings {
return this.section(fname);
}
/**
* Produce a summary description of the setting and how it was
* derived.
*/
public describe(): AppSettingDescription {
return {
name: this.name,
value: (this._info?.query.censor && this._value !== undefined) ? '*****' : this._value,
foundInEnvVar: this._info?.envVar,
wouldFindInEnvVar: this._info?.query.preferredEnvVar || getEnvVarsFromQuery(this._info?.query)[0],
usedDefault: this._value !== undefined && this._info !== undefined && !this._info?.found,
};
}
/**
* As for describe(), but include all nested settings also.
* Used dotted notation for setting names. Omit settings that
* are undefined and without useful information about how they
* might be defined.
*/
public describeAll(): AppSettingDescription[] {
const inv: AppSettingDescription[] = [];
inv.push(this.describe());
if (this._children) {
for (const child of Object.values(this._children)) {
for (const item of child.describeAll()) {
inv.push({...item, name: this.name + '.' + item.name});
}
}
}
return inv.filter(item => item.value !== undefined ||
item.wouldFindInEnvVar !== undefined ||
item.usedDefault);
}
}
/**
* A global object for Grist application settings.
*/
export const appSettings = new AppSettings('grist');
/**
* Hints for how to define a setting, including possible
* environment variables and default values.
*/
export interface AppSettingQuery {
envVar: string|string[]; // environment variable(s) to check.
preferredEnvVar?: string; // "Canonical" environment variable to suggest.
// Should be in envVar (though this is not checked).
defaultValue?: JSONValue; // value to use if variable(s) unavailable.
censor?: boolean; // should the value of the setting be obscured when printed.
}
/**
* Result of a query specifying whether the setting
* was found, and if so in what environment variable, and using
* what query.
*/
export interface AppSettingQueryResult {
envVar?: string;
found: boolean;
query: AppSettingQuery;
}
/**
* Output of AppSettings.describe().
*/
interface AppSettingDescription {
name: string; // name of the setting.
value?: JSONValue; // value of the setting, if available.
foundInEnvVar?: string; // environment variable the setting was read from, if available.
wouldFindInEnvVar?: string; // environment variable that would be checked for the setting.
usedDefault: boolean; // whether a default value was used for the setting.
}
// Helper function to normalize the AppSettingQuery.envVar list.
function getEnvVarsFromQuery(q?: AppSettingQuery): string[] {
if (!q) { return []; }
return Array.isArray(q.envVar) ? q.envVar : [q.envVar];
}
// Keep app settings JSON-like, in case later we decide to load them from
// a JSON source.
type JSONValue = string | number | boolean | null | { [member: string]: JSONValue } | JSONValue[];

@ -527,7 +527,7 @@ export class DocManager extends EventEmitter {
const doc = await this._getDoc(docSession, docName);
// Get URL for document for use with SELF_HYPERLINK().
const docUrl = doc && await this._getDocUrl(doc);
return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode, doc});
return new ActiveDoc(this, docName, {docUrl, safeMode, doc});
}
/**

@ -42,7 +42,7 @@ export class DocStorageManager implements IDocStorageManager {
private _comm?: Comm, gristServer?: GristServer) {
// If we have a way to communicate with clients, watch the docsRoot for changes.
this._watcher = null;
this._shell = (gristServer && gristServer.create.Shell()) || {
this._shell = gristServer?.create.Shell?.() || {
moveItemToTrash() { throw new Error('Unable to move document to trash'); },
showItemInFolder() { throw new Error('Unable to show item in folder'); }
};

@ -18,6 +18,7 @@ import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {Housekeeper} from 'app/gen-server/lib/Housekeeper';
import {Usage} from 'app/gen-server/lib/Usage';
import {attachAppEndpoint} from 'app/server/lib/AppEndpoint';
import {appSettings} from 'app/server/lib/AppSettings';
import {addRequestUser, getUser, getUserId, isSingleUserMode,
redirectToLoginUnconditionally} from 'app/server/lib/Authorizer';
import {redirectToLogin, RequestWithLogin, signInStatusMiddleware} from 'app/server/lib/Authorizer';
@ -127,7 +128,7 @@ export class FlexServer implements GristServer {
private _internalPermitStore: IPermitStore; // store for permits that stay within our servers
private _externalPermitStore: IPermitStore; // store for permits that pass through outside servers
private _disabled: boolean = false;
private _disableS3: boolean = false;
private _disableExternalStorage: boolean = false;
private _healthy: boolean = true; // becomes false if a serious error has occurred and
// server cannot do its work.
private _healthCheckCounter: number = 0;
@ -1006,21 +1007,21 @@ export class FlexServer implements GristServer {
await this.loadConfig();
this.addComm();
await this.create.configure?.();
if (!isSingleUserMode()) {
if (!process.env.GRIST_DOCS_S3_BUCKET || process.env.GRIST_DISABLE_S3 === 'true') {
this._disableS3 = true;
const externalStorage = appSettings.section('externalStorage');
const haveExternalStorage = Object.values(externalStorage.nested)
.some(storage => storage.flag('active').getAsBool());
const disabled = externalStorage.flag('disable')
.read({ envVar: 'GRIST_DISABLE_S3' }).getAsBool();
if (disabled || !haveExternalStorage) {
this._disableExternalStorage = true;
externalStorage.flag('active').set(false);
}
for (const [key, val] of Object.entries(this.create.configurationOptions())) {
this.info.push([key, val]);
}
if (this._disableS3) {
this.info.push(['s3', 'disabled']);
}
const workers = this._docWorkerMap;
const docWorkerId = await this._addSelfAsWorker(workers);
const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableS3, workers,
const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableExternalStorage, workers,
this._dbManager, this.create);
this._storageManager = storageManager;
} else {
@ -1065,11 +1066,11 @@ export class FlexServer implements GristServer {
}
}
public disableS3() {
public disableExternalStorage() {
if (this.deps.has('doc')) {
throw new Error('disableS3 called too late');
throw new Error('disableExternalStorage called too late');
}
this._disableS3 = true;
this._disableExternalStorage = true;
}
public addAccountPage() {
@ -1203,6 +1204,14 @@ export class FlexServer implements GristServer {
for (const [label, value] of this.info) {
log.info("== %s: %s", label, value);
}
for (const item of appSettings.describeAll()) {
const txt =
((item.value !== undefined) ? String(item.value) : '-') +
(item.foundInEnvVar ? ` [${item.foundInEnvVar}]` : '') +
(item.usedDefault ? ' [default]' : '') +
((item.wouldFindInEnvVar && !item.foundInEnvVar) ? ` [${item.wouldFindInEnvVar}]` : '');
log.info("== %s: %s", item.name, txt);
}
}
public async start() {

@ -1,7 +1,5 @@
import {Document} from 'app/gen-server/entity/Document';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {DocManager} from 'app/server/lib/DocManager';
import {ExternalStorage} from 'app/server/lib/ExternalStorage';
import {GristServer} from 'app/server/lib/GristServer';
import {IBilling} from 'app/server/lib/IBilling';
@ -15,7 +13,7 @@ export interface ICreate {
Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
Shell(): IShell|undefined;
Shell?(): IShell; // relevant to electron version of Grist only.
// Create a space to store files externally, for storing either:
// - documents. This store should be versioned, and can be eventually consistent.
@ -24,12 +22,11 @@ export interface ICreate {
// should not interfere with each other.
ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage | undefined;
ActiveDoc(docManager: DocManager, docName: string, options: ICreateActiveDocOptions): ActiveDoc;
NSandbox(options: ISandboxCreationOptions): ISandbox;
sessionSecret(): string;
// Get configuration information to show at start-up.
configurationOptions(): {[key: string]: any};
// Check configuration of the app early enough to show on startup.
configure?(): Promise<void>;
// Return a string containing 1 or more HTML tags to insert into the head element of every
// static page.
getExtraHeadHtml?(): string;
@ -42,7 +39,7 @@ export interface ICreateActiveDocOptions {
}
export interface ICreateStorageOptions {
check(): Record<string, string>|undefined;
check(): boolean;
create(purpose: 'doc'|'meta', extraPrefix: string): ExternalStorage|undefined;
}
@ -75,20 +72,14 @@ export function makeSimpleCreator(opts: {
deleteUser() { throw new Error('deleteUser unavailable'); },
};
},
Shell() {
return {
moveItemToTrash() { throw new Error('moveToTrash unavailable'); },
showItemInFolder() { throw new Error('showItemInFolder unavailable'); }
};
},
ExternalStorage(purpose, extraPrefix) {
for (const storage of opts.storage || []) {
const config = storage.check();
if (config) { return storage.create(purpose, extraPrefix); }
if (storage.check()) {
return storage.create(purpose, extraPrefix);
}
}
return undefined;
},
ActiveDoc(docManager, docName, options) { return new ActiveDoc(docManager, docName, options); },
NSandbox(options) {
return createSandbox('unsandboxed', options);
},
@ -99,12 +90,10 @@ export function makeSimpleCreator(opts: {
}
return secret;
},
configurationOptions() {
async configure() {
for (const storage of opts.storage || []) {
const config = storage.check();
if (config) { return config; }
if (storage.check()) { break; }
}
return {};
},
getExtraHeadHtml() {
let customHeadHtmlSnippet = '';

@ -34,7 +34,7 @@ interface ServerOptions extends FlexServerOptions {
logToConsole?: boolean; // If set, messages logged to console (default: false)
// (but if options are not given at all in call to main,
// logToConsole is set to true)
s3?: boolean; // If set, documents saved to s3 (default is to check environment
externalStorage?: boolean; // If set, documents saved to external storage such as s3 (default is to check environment
// variables, which get set in various ways in dev/test entry points)
}
@ -59,7 +59,7 @@ export async function main(port: number, serverTypes: ServerType[],
}
if (options.logToConsole) { server.addLogging(); }
if (options.s3 === false) { server.disableS3(); }
if (options.externalStorage === false) { server.disableExternalStorage(); }
await server.loadConfig();
if (includeDocs) {

Loading…
Cancel
Save