diff --git a/app/common/DocSnapshot.ts b/app/common/DocSnapshot.ts index 27e49f49..b2ff85ca 100644 --- a/app/common/DocSnapshot.ts +++ b/app/common/DocSnapshot.ts @@ -35,3 +35,35 @@ export interface DocSnapshot extends ObjSnapshotWithMetadata { export interface DocSnapshots { snapshots: DocSnapshot[]; } + +/** + * Metadata format for external storage like S3 and Azure. + * The only difference is that external metadata values must be strings. + * + * For S3, there are restrictions on total length of metadata (2 KB). + * See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#UserMetadata + */ +type ExternalMetadata = Record; + +/** + * Convert metadata from internal Grist format to external storage format (string values). + */ +export function toExternalMetadata(metadata: ObjMetadata): ExternalMetadata { + const result: ExternalMetadata = {}; + for (const [key, val] of Object.entries(metadata)) { + if (val !== undefined) { result[key] = String(val); } + } + return result; +} + +/** + * Select metadata controlled by Grist, and convert to expected formats. + */ +export function toGristMetadata(metadata: ExternalMetadata): ObjMetadata { + const result: ObjMetadata = {}; + for (const key of ['t', 'tz', 'h', 'label'] as const) { + if (metadata[key]) { result[key] = metadata[key]; } + } + if (metadata.n) { result.n = parseInt(metadata.n, 10); } + return result; +} diff --git a/app/common/asyncIterators.ts b/app/common/asyncIterators.ts new file mode 100644 index 00000000..55d78e38 --- /dev/null +++ b/app/common/asyncIterators.ts @@ -0,0 +1,25 @@ +/** + * Just some basic utilities for async generators that should really be part of the language or lodash or something. + */ + +export async function* asyncFilter(it: AsyncIterableIterator, pred: (x: T) => boolean): AsyncIterableIterator { + for await (const x of it) { + if (pred(x)) { + yield x; + } + } +} + +export async function* asyncMap(it: AsyncIterableIterator, mapper: (x: T) => R): AsyncIterableIterator { + for await (const x of it) { + yield mapper(x); + } +} + +export async function toArray(it: AsyncIterableIterator): Promise { + const result = []; + for await (const x of it) { + result.push(x); + } + return result; +} diff --git a/app/server/lib/HostedStorageManager.ts b/app/server/lib/HostedStorageManager.ts index 31cbcb67..c76162db 100644 --- a/app/server/lib/HostedStorageManager.ts +++ b/app/server/lib/HostedStorageManager.ts @@ -45,12 +45,15 @@ function checkValidDocId(docId: string): void { } } -interface HostedStorageOptions { +export interface HostedStorageOptions { secondsBeforePush: number; secondsBeforeFirstRetry: number; pushDocUpdateTimes: boolean; - testExternalStorageDoc?: ExternalStorage; - testExternalStorageMeta?: ExternalStorage; + // A function returning the core ExternalStorage implementation, + // which may then be wrapped in additional layer(s) of ExternalStorage. + // See ICreate.ExternalStorage. + // Uses S3 by default in hosted Grist. + innerExternalStorageCreate?: (bucket: string) => ExternalStorage; } const defaultOptions: HostedStorageOptions = { @@ -132,8 +135,8 @@ export class HostedStorageManager implements IDocStorageManager { ) { // We store documents either in a test store, or in an s3 store // at s3:///.grist - const externalStoreDoc = options.testExternalStorageDoc || - (this._disableS3 ? undefined : create.ExternalStorage('doc', extraS3Prefix)); + const externalStoreDoc = this._disableS3 ? undefined : + create.ExternalStorage('doc', extraS3Prefix, options.innerExternalStorageCreate); if (!externalStoreDoc) { this._disableS3 = true; } const secondsBeforePush = options.secondsBeforePush; if (options.pushDocUpdateTimes) { @@ -154,7 +157,7 @@ export class HostedStorageManager implements IDocStorageManager { this._ext = this._getChecksummedExternalStorage('doc', this._baseStore, this._latestVersions, options); - const baseStoreMeta = options.testExternalStorageMeta || create.ExternalStorage('meta', extraS3Prefix); + const baseStoreMeta = create.ExternalStorage('meta', extraS3Prefix, options.innerExternalStorageCreate); if (!baseStoreMeta) { throw new Error('bug: external storage should be created for "meta" if it is created for "doc"'); } diff --git a/app/server/lib/ICreate.ts b/app/server/lib/ICreate.ts index 44f90ea9..0b0874ed 100644 --- a/app/server/lib/ICreate.ts +++ b/app/server/lib/ICreate.ts @@ -20,7 +20,14 @@ export interface ICreate { // - meta. This store need not be versioned, and can be eventually consistent. // For test purposes an extra prefix may be supplied. Stores with different prefixes // should not interfere with each other. - ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage|undefined; + // innerCreate should be a function returning the core ExternalStorage implementation, + // which this method may wrap in additional layer(s) of ExternalStorage. + // Uses S3 by default in hosted Grist. + ExternalStorage( + purpose: 'doc' | 'meta', + testExtraPrefix: string, + innerCreate?: (bucket: string) => ExternalStorage + ): ExternalStorage | undefined; ActiveDoc(docManager: DocManager, docName: string, options: ICreateActiveDocOptions): ActiveDoc; NSandbox(options: ISandboxCreationOptions): ISandbox;