(core) Add AzureExternalStorage

Summary:
Adds a new implementation of the interface ExternalStorage that works for Azure Blob Storage as an alternative to S3, for a specific self-hosting case.

Tweaks HostedStorageManager and ICreate to allow configuring different core implementations of ExternalStorage.

Followup tasks:

- Make this code available to self hosters, possibly by making it open source.
- Add an env var or other config option to specify the preferred type of storage. Currently using the var `AZURE_STORAGE_CONNECTION_STRING` to know how to connect to Azure when requested, but that choice still only lives in test code.

Test Plan: Generalized HostedStorageManager and ExternalStorage tests to test the new AzureExternalStorage alongside S3ExternalStorage. The HostedStorageManager tests also now test the 'cached' in-memory test storage in a way that's closer to the real storage methods.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3413

app/common/DocSnapshot.ts

@@ -35,3 +35,35 @@ export interface DocSnapshot extends ObjSnapshotWithMetadata {
 export interface DocSnapshots {
   snapshots: DocSnapshot[];
 }
+
+/**
+ * Metadata format for external storage like S3 and Azure.
+ * The only difference is that external metadata values must be strings.
+ *
+ * For S3, there are restrictions on total length of metadata (2 KB).
+ * See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#UserMetadata
+ */
+type ExternalMetadata = Record<string, string>;
+
+/**
+ * Convert metadata from internal Grist format to external storage format (string values).
+ */
+export function toExternalMetadata(metadata: ObjMetadata): ExternalMetadata {
+  const result: ExternalMetadata = {};
+  for (const [key, val] of Object.entries(metadata)) {
+    if (val !== undefined) { result[key] = String(val); }
+  }
+  return result;
+}
+
+/**
+ * Select metadata controlled by Grist, and convert to expected formats.
+ */
+export function toGristMetadata(metadata: ExternalMetadata): ObjMetadata {
+  const result: ObjMetadata = {};
+  for (const key of ['t', 'tz', 'h', 'label'] as const) {
+    if (metadata[key]) { result[key] = metadata[key]; }
+  }
+  if (metadata.n) { result.n = parseInt(metadata.n, 10); }
+  return result;
+}

app/common/asyncIterators.ts

@@ -0,0 +1,25 @@
+/**
+ * Just some basic utilities for async generators that should really be part of the language or lodash or something.
+ */
+
+export async function* asyncFilter<T>(it: AsyncIterableIterator<T>, pred: (x: T) => boolean): AsyncIterableIterator<T> {
+  for await (const x of it) {
+    if (pred(x)) {
+      yield x;
+    }
+  }
+}
+
+export async function* asyncMap<T, R>(it: AsyncIterableIterator<T>, mapper: (x: T) => R): AsyncIterableIterator<R> {
+  for await (const x of it) {
+    yield mapper(x);
+  }
+}
+
+export async function toArray<T>(it: AsyncIterableIterator<T>): Promise<T[]> {
+  const result = [];
+  for await (const x of it) {
+    result.push(x);
+  }
+  return result;
+}

app/server/lib/HostedStorageManager.ts

@@ -45,12 +45,15 @@ function checkValidDocId(docId: string): void {
   }
 }
 
-interface HostedStorageOptions {
+export interface HostedStorageOptions {
   secondsBeforePush: number;
   secondsBeforeFirstRetry: number;
   pushDocUpdateTimes: boolean;
-  testExternalStorageDoc?: ExternalStorage;
-  testExternalStorageMeta?: ExternalStorage;
+  // A function returning the core ExternalStorage implementation,
+  // which may then be wrapped in additional layer(s) of ExternalStorage.
+  // See ICreate.ExternalStorage.
+  // Uses S3 by default in hosted Grist.
+  innerExternalStorageCreate?: (bucket: string) => ExternalStorage;
 }
 
 const defaultOptions: HostedStorageOptions = {
@@ -132,8 +135,8 @@ export class HostedStorageManager implements IDocStorageManager {
   ) {
     // We store documents either in a test store, or in an s3 store
     // at s3://<s3Bucket>/<s3Prefix><docId>.grist
-    const externalStoreDoc = options.testExternalStorageDoc ||
-      (this._disableS3 ? undefined : create.ExternalStorage('doc', extraS3Prefix));
+    const externalStoreDoc = this._disableS3 ? undefined :
+      create.ExternalStorage('doc', extraS3Prefix, options.innerExternalStorageCreate);
     if (!externalStoreDoc) { this._disableS3 = true; }
     const secondsBeforePush = options.secondsBeforePush;
     if (options.pushDocUpdateTimes) {
@@ -154,7 +157,7 @@ export class HostedStorageManager implements IDocStorageManager {
       this._ext = this._getChecksummedExternalStorage('doc', this._baseStore,
                                                       this._latestVersions, options);
 
-      const baseStoreMeta = options.testExternalStorageMeta || create.ExternalStorage('meta', extraS3Prefix);
+      const baseStoreMeta = create.ExternalStorage('meta', extraS3Prefix, options.innerExternalStorageCreate);
       if (!baseStoreMeta) {
         throw new Error('bug: external storage should be created for "meta" if it is created for "doc"');
       }

app/server/lib/ICreate.ts

@@ -20,7 +20,14 @@ export interface ICreate {
   //  - meta. This store need not be versioned, and can be eventually consistent.
   // For test purposes an extra prefix may be supplied.  Stores with different prefixes
   // should not interfere with each other.
-  ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage|undefined;
+  // innerCreate should be a function returning the core ExternalStorage implementation,
+  // which this method may wrap in additional layer(s) of ExternalStorage.
+  // Uses S3 by default in hosted Grist.
+  ExternalStorage(
+    purpose: 'doc' | 'meta',
+    testExtraPrefix: string,
+    innerCreate?: (bucket: string) => ExternalStorage
+  ): ExternalStorage | undefined;
 
   ActiveDoc(docManager: DocManager, docName: string, options: ICreateActiveDocOptions): ActiveDoc;
   NSandbox(options: ISandboxCreationOptions): ISandbox;