(core) revamp snapshot inventory

Summary:
Deliberate changes:
 * save snapshots to s3 prior to migrations.
 * label migration snapshots in s3 metadata.
 * avoid pruning migration snapshots for a month.

Opportunistic changes:
 * Associate document timezone with snapshots, so pruning can respect timezones.
 * Associate actionHash/Num with snapshots.
 * Record time of last change in snapshots (rather than just s3 upload time, which could be a while later).

This ended up being a biggish change, because there was nowhere ideal to put tags (list of possibilities in diff).

Test Plan: added tests

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2646

app/common/KeyedMutex.ts

@@ -0,0 +1,46 @@
+import { Mutex, MutexInterface } from 'async-mutex';
+
+/**
+ * A per-key mutex.  It has the same interface as Mutex, but with an extra key supplied.
+ * Maintains an independent mutex for each key on need.
+ */
+export class KeyedMutex {
+  private _mutexes = new Map<string, Mutex>();
+
+  public async acquire(key: string): Promise<MutexInterface.Releaser> {
+    // Create a new mutex if we need one.
+    if (!this._mutexes.has(key)) {
+      this._mutexes.set(key, new Mutex());
+    }
+    const mutex = this._mutexes.get(key)!
+    const unlock = await mutex.acquire();
+    return () => {
+      unlock();
+      // After unlocking, clean-up the mutex if it is no longer needed.
+      // unlock() leaves the mutex locked if anyone has been waiting for it.
+      if (!mutex.isLocked()) {
+        this._mutexes.delete(key);
+      }
+    };
+  }
+
+  public async runExclusive<T>(key: string, callback: MutexInterface.Worker<T>): Promise<T> {
+    const unlock = await this.acquire(key);
+    try {
+      return callback();
+    } finally {
+      unlock();
+    }
+  }
+
+  public isLocked(key: string): boolean {
+    const mutex = this._mutexes.get(key);
+    if (!mutex) { return false; }
+    return mutex.isLocked();
+  }
+
+  // Check how many mutexes are in use.
+  public get size(): number {
+    return this._mutexes.size;
+  }
+}