mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
39eb042ff1
Skipping the redis checksum mismatch is now generalized. A warning is logged when we see a mismatch.
1039 lines
40 KiB
TypeScript
1039 lines
40 KiB
TypeScript
import {ErrorOrValue, freezeError, mapGetOrSet, MapWithTTL} from 'app/common/AsyncCreate';
|
|
import {ObjMetadata, ObjSnapshot, ObjSnapshotWithMetadata} from 'app/common/DocSnapshot';
|
|
import {SCHEMA_VERSION} from 'app/common/schema';
|
|
import {DocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
|
|
import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
|
|
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
|
|
import {create} from 'app/server/lib/create';
|
|
import {DocManager} from 'app/server/lib/DocManager';
|
|
import {makeExceptionalDocSession} from 'app/server/lib/DocSession';
|
|
import {DELETED_TOKEN, ExternalStorage, wrapWithKeyMappedStorage} from 'app/server/lib/ExternalStorage';
|
|
import {createDummyGristServer} from 'app/server/lib/GristServer';
|
|
import {
|
|
BackupEvent,
|
|
backupSqliteDatabase,
|
|
HostedStorageManager,
|
|
HostedStorageOptions
|
|
} from 'app/server/lib/HostedStorageManager';
|
|
import log from 'app/server/lib/log';
|
|
import {SQLiteDB} from 'app/server/lib/SQLiteDB';
|
|
import * as bluebird from 'bluebird';
|
|
import {assert} from 'chai';
|
|
import * as fse from 'fs-extra';
|
|
import * as path from 'path';
|
|
import {createClient, RedisClient} from 'redis';
|
|
import * as sinon from 'sinon';
|
|
import {createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed';
|
|
import {createTmpDir, getGlobalPluginManager} from 'test/server/docTools';
|
|
import {EnvironmentSnapshot, setTmpLogLevel, useFixtureDoc} from 'test/server/testUtils';
|
|
import {waitForIt} from 'test/server/wait';
|
|
import uuidv4 from "uuid/v4";
|
|
|
|
bluebird.promisifyAll(RedisClient.prototype);
|
|
|
|
/**
|
|
* An in-memory store, for testing.
|
|
*/
|
|
class SimpleExternalStorage implements ExternalStorage {
|
|
protected _version = new Map<string, ObjSnapshot[]>();
|
|
private _nextId: number = 1;
|
|
private _memory = new Map<string, Buffer>();
|
|
private _metadata = new Map<string, ObjSnapshotWithMetadata>();
|
|
|
|
public constructor(public readonly label: string) {}
|
|
|
|
public async exists(key: string, snapshotId?: string): Promise<boolean> {
|
|
if (snapshotId) {
|
|
// Should check snapshotId is associated with key, but don't need to be too
|
|
// fussy this mock.
|
|
return this._memory.has(snapshotId);
|
|
}
|
|
return this._version.has(key);
|
|
}
|
|
|
|
public async head(key: string, snapshotId?: string) {
|
|
snapshotId = snapshotId || this._version.get(key)?.[0]?.snapshotId;
|
|
if (!snapshotId) { return null; }
|
|
return this._metadata.get(snapshotId) || null;
|
|
}
|
|
|
|
public async upload(key: string, fname: string, metadata?: ObjMetadata) {
|
|
const data = await fse.readFile(fname);
|
|
const id = `block-${this._nextId}`;
|
|
this._nextId++;
|
|
this._memory.set(id, data);
|
|
const info: ObjSnapshotWithMetadata = {
|
|
snapshotId: id,
|
|
lastModified: new Date().toISOString(),
|
|
};
|
|
this._metadata.set(id, {...info, ...metadata && {metadata}});
|
|
const versions = this._version.get(key) || [];
|
|
versions.unshift(info);
|
|
this._version.set(key, versions);
|
|
return id;
|
|
}
|
|
|
|
public async remove(key: string, snapshotIds?: string[]) {
|
|
const versions = this._version.get(key);
|
|
if (!versions) { return; }
|
|
if (!snapshotIds) {
|
|
for (const version of versions) {
|
|
this._memory.delete(version.snapshotId);
|
|
this._metadata.delete(version.snapshotId);
|
|
}
|
|
this._version.delete(key);
|
|
} else {
|
|
for (const snapshotId of snapshotIds) {
|
|
this._memory.delete(snapshotId);
|
|
this._metadata.delete(snapshotId);
|
|
}
|
|
const blockList = new Set(snapshotIds);
|
|
this._version.set(key, (this._version.get(key) || []).filter(v => !blockList.has(v.snapshotId)));
|
|
}
|
|
}
|
|
|
|
public async download(key: string, fname: string, snapshotId?: string) {
|
|
const versions = this._version.get(key);
|
|
if (!versions) { throw new Error('oopsie key not found'); }
|
|
if (snapshotId) {
|
|
if (!versions.find(v => v.snapshotId === snapshotId)) {
|
|
throw new Error('version not recognized');
|
|
}
|
|
} else {
|
|
snapshotId = versions[0].snapshotId;
|
|
}
|
|
if (!snapshotId) { throw new Error('version not found'); }
|
|
const data = this._memory.get(snapshotId);
|
|
if (!data) { throw new Error('version data not found'); }
|
|
await fse.writeFile(fname, data);
|
|
return snapshotId;
|
|
}
|
|
|
|
public async versions(key: string) {
|
|
return this._version.get(key) || [];
|
|
}
|
|
|
|
public url(key: string): string {
|
|
return `simple://test/${this.label}/${key}`;
|
|
}
|
|
|
|
public isFatalError(err: any): boolean {
|
|
return !String(err).includes('oopsie');
|
|
}
|
|
|
|
public async close() {
|
|
// nothing to do
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A wrapper around an external store, that deliberately gives stale values for the
|
|
* `exists`, `download`, and `versions` methods.
|
|
*/
|
|
class CachedExternalStorage implements ExternalStorage {
|
|
private _cachedExists: MapWithTTL<string, Promise<ErrorOrValue<boolean>>>;
|
|
private _cachedHead: MapWithTTL<string, Promise<ErrorOrValue<ObjSnapshotWithMetadata|null>>>;
|
|
private _cachedDownload: MapWithTTL<string, Promise<ErrorOrValue<[string, Buffer]>>>;
|
|
private _cachedVersions: MapWithTTL<string, Promise<ErrorOrValue<ObjSnapshot[]>>>;
|
|
|
|
constructor(private _ext: ExternalStorage, ttlMs: number) {
|
|
this._cachedExists = new MapWithTTL(ttlMs);
|
|
this._cachedHead = new MapWithTTL(ttlMs);
|
|
this._cachedDownload = new MapWithTTL(ttlMs);
|
|
this._cachedVersions = new MapWithTTL(ttlMs);
|
|
}
|
|
|
|
public async exists(key: string, snapshotId?: string) {
|
|
const result = await mapGetOrSet(this._cachedExists, `${key}${snapshotId}`, () => {
|
|
return freezeError(this._ext.exists(key, snapshotId));
|
|
});
|
|
return result.unfreeze();
|
|
}
|
|
|
|
public async head(key: string, snapshotId?: string) {
|
|
const result = await mapGetOrSet(this._cachedHead, `${key}${snapshotId}`, () => {
|
|
return freezeError(this._ext.head(key, snapshotId));
|
|
});
|
|
return result.unfreeze();
|
|
}
|
|
|
|
public async upload(key: string, fname: string, metadata?: ObjMetadata) {
|
|
return this._ext.upload(key, fname, metadata);
|
|
}
|
|
|
|
public async remove(key: string, snapshotIds?: string[]) {
|
|
return this._ext.remove(key, snapshotIds);
|
|
}
|
|
|
|
public async download(key: string, fname: string, snapshotId?: string): Promise<string> {
|
|
const result = await mapGetOrSet(this._cachedDownload, `${key}${snapshotId}`, () => {
|
|
const altFname = fname + uuidv4();
|
|
return freezeError(
|
|
this._ext.download(key, altFname, snapshotId).then(async (v) => {
|
|
return [v, await fse.readFile(altFname)] as [string, Buffer];
|
|
})
|
|
);
|
|
});
|
|
try {
|
|
const [downloadedSnapshotId, txt] = await result.unfreeze();
|
|
await fse.writeFile(fname, txt);
|
|
return downloadedSnapshotId;
|
|
} catch (e) {
|
|
await fse.writeFile(fname, 'put some junk here to simulate unclean failure');
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
public async versions(key: string) {
|
|
const result = await mapGetOrSet(this._cachedVersions, key, () => {
|
|
return freezeError(this._ext.versions(key));
|
|
});
|
|
return result.unfreeze();
|
|
}
|
|
|
|
public url(key: string): string {
|
|
return this._ext.url(key);
|
|
}
|
|
|
|
public isFatalError(err: any): boolean {
|
|
return this._ext.isFatalError(err);
|
|
}
|
|
|
|
public async close() {
|
|
// nothing to do
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A wrapper that slows down responses from a store.
|
|
*/
|
|
class SlowExternalStorage implements ExternalStorage {
|
|
constructor(private _ext: ExternalStorage, private _delayMs: number) {}
|
|
|
|
public async exists(key: string, snapshotId?: string) {
|
|
await bluebird.delay(this._delayMs);
|
|
return this._ext.exists(key, snapshotId);
|
|
}
|
|
|
|
public async head(key: string, snapshotId?: string) {
|
|
await bluebird.delay(this._delayMs);
|
|
return this._ext.head(key, snapshotId);
|
|
}
|
|
|
|
public async upload(key: string, fname: string, metadata?: ObjMetadata) {
|
|
await bluebird.delay(this._delayMs);
|
|
return this._ext.upload(key, fname, metadata);
|
|
}
|
|
|
|
public async remove(key: string, snapshotIds?: string[]) {
|
|
await bluebird.delay(this._delayMs);
|
|
return this._ext.remove(key, snapshotIds);
|
|
}
|
|
|
|
public async download(key: string, fname: string, snapshotId?: string): Promise<string> {
|
|
await bluebird.delay(this._delayMs);
|
|
return this._ext.download(key, fname, snapshotId);
|
|
}
|
|
|
|
public async versions(key: string) {
|
|
await bluebird.delay(this._delayMs);
|
|
return this._ext.versions(key);
|
|
}
|
|
|
|
public url(key: string): string {
|
|
return this._ext.url(key);
|
|
}
|
|
|
|
public isFatalError(err: any): boolean {
|
|
return this._ext.isFatalError(err);
|
|
}
|
|
|
|
public async close() {
|
|
// nothing to do
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* A document store representing a doc worker's local store, for testing.
|
|
* Uses TEST_S3_BUCKET and TEST_S3_PREFIX. Objects in test bucket should be set up
|
|
* to be deleted after a short period. Since we don't attempt to garbage collect
|
|
* within the unit test. s3://grist-docs-test/unit-tests/... is set up that way.
|
|
*/
|
|
class TestStore {
|
|
public docManager: DocManager;
|
|
public storageManager: HostedStorageManager;
|
|
private _active: boolean = false; // True if the simulated doc worker is started.
|
|
private _extraPrefix = uuidv4(); // Extra prefix in S3 (unique to this test).
|
|
|
|
public constructor(
|
|
private _localDirectory: string,
|
|
private _workerId: string,
|
|
private _workers: DocWorkerMap,
|
|
private _externalStorageCreate: (purpose: 'doc'|'meta', extraPrefix: string) => ExternalStorage|undefined) {
|
|
}
|
|
|
|
public async run<T>(fn: () => Promise<T>): Promise<T> {
|
|
await this.begin();
|
|
let result;
|
|
try {
|
|
result = await fn();
|
|
} finally {
|
|
await this.end();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Simulates doc worker startup.
|
|
public async begin() {
|
|
await this.end();
|
|
this._active = true;
|
|
const dbManager = new HomeDBManager();
|
|
await dbManager.connect();
|
|
await dbManager.initializeSpecialIds();
|
|
const options: HostedStorageOptions = {
|
|
secondsBeforePush: 0.5,
|
|
secondsBeforeFirstRetry: 3, // rumors online suggest delays of 10-11 secs
|
|
// are not super-unusual.
|
|
pushDocUpdateTimes: false,
|
|
externalStorageCreator: (purpose) => {
|
|
const result = this._externalStorageCreate(purpose, this._extraPrefix);
|
|
if (!result) { throw new Error('no storage'); }
|
|
return result;
|
|
}
|
|
};
|
|
const storageManager = new HostedStorageManager(this._localDirectory,
|
|
this._workerId,
|
|
false,
|
|
this._workers,
|
|
dbManager,
|
|
create,
|
|
options);
|
|
this.storageManager = storageManager;
|
|
this.docManager = new DocManager(storageManager, await getGlobalPluginManager(),
|
|
dbManager, {
|
|
...createDummyGristServer(),
|
|
getStorageManager() { return storageManager; },
|
|
});
|
|
}
|
|
|
|
// Simulates doc worker shutdown. Closes all open documents.
|
|
public async end() {
|
|
if (this._active) {
|
|
await this.docManager.shutdownAll();
|
|
}
|
|
this._active = false;
|
|
}
|
|
|
|
// Close a single doc. The server does this for docs that are not open by
|
|
// any client.
|
|
public async closeDoc(doc: ActiveDoc) {
|
|
await doc.shutdown();
|
|
}
|
|
|
|
// Waits for any required S3 pushes to have completed.
|
|
public async waitForUpdates(): Promise<boolean> {
|
|
for (let i = 0; i < 50; i++) {
|
|
if (!this.storageManager.needsUpdate()) {
|
|
return true;
|
|
}
|
|
await bluebird.delay(100);
|
|
}
|
|
log.error("waitForUpdates failed");
|
|
return false;
|
|
}
|
|
|
|
// Wipes the doc worker's local document store.
|
|
public async removeAll(): Promise<void> {
|
|
const fnames = await fse.readdir(this._localDirectory);
|
|
await Promise.all(fnames.map(fname => {
|
|
return fse.remove(path.join(this._localDirectory, fname));
|
|
}));
|
|
}
|
|
|
|
public getDocPath(docId: string) {
|
|
return path.join(this._localDirectory, `${docId}.grist`);
|
|
}
|
|
}
|
|
|
|
|
|
describe('HostedStorageManager', function() {
|
|
|
|
setTmpLogLevel('info'); // allow info messages for this test since failures are hard to replicate
|
|
this.timeout(60000); // s3 can be slow
|
|
|
|
const docSession = makeExceptionalDocSession('system');
|
|
|
|
before(async function() {
|
|
setUpDB(this);
|
|
await createInitialDb();
|
|
});
|
|
|
|
after(async function() {
|
|
await removeConnection();
|
|
});
|
|
|
|
for (const storage of ['azure', 's3', 'minio', 'cached'] as const) {
|
|
describe(storage, function() {
|
|
|
|
const sandbox = sinon.createSandbox();
|
|
let oldEnv: EnvironmentSnapshot;
|
|
|
|
const workerId = 'dw17';
|
|
let cli: RedisClient;
|
|
let store: TestStore;
|
|
let workers: DocWorkerMap;
|
|
let tmpDir: string;
|
|
|
|
before(async function() {
|
|
if (!process.env.TEST_REDIS_URL) { this.skip(); return; }
|
|
cli = createClient(process.env.TEST_REDIS_URL);
|
|
oldEnv = new EnvironmentSnapshot();
|
|
await cli.flushdbAsync();
|
|
workers = new DocWorkerMap([cli]);
|
|
await workers.addWorker({
|
|
id: workerId,
|
|
publicUrl: 'notset',
|
|
internalUrl: 'notset',
|
|
});
|
|
await workers.setWorkerAvailability(workerId, true);
|
|
|
|
await workers.assignDocWorker('Hello');
|
|
await workers.assignDocWorker('Hello2');
|
|
|
|
tmpDir = await createTmpDir();
|
|
|
|
let externalStorageCreate: (purpose: 'doc'|'meta', extraPrefix: string) => ExternalStorage|undefined;
|
|
function requireStorage<T>(storage: T|undefined): T {
|
|
if (storage === undefined) { throw new Error('storage not found'); }
|
|
return storage;
|
|
}
|
|
switch (storage) {
|
|
case 'cached': {
|
|
// Make an in-memory store that is slow and aggressively cached.
|
|
// This tickles a lot of cases that occasionally happen with s3.
|
|
let ext: ExternalStorage = new SimpleExternalStorage("bucket");
|
|
ext = new CachedExternalStorage(ext, 1000);
|
|
ext = new SlowExternalStorage(ext, 250);
|
|
// Everything is stored in fields of these objects, so the tests mustn't recreate them repeatedly.
|
|
externalStorageCreate = (purpose) => wrapWithKeyMappedStorage(ext, {purpose, basePrefix: 'prefix'});
|
|
break;
|
|
}
|
|
case 'azure':
|
|
if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
|
|
this.skip();
|
|
}
|
|
externalStorageCreate = requireStorage(create.getStorageOptions?.('azure')?.create);
|
|
break;
|
|
case 'minio':
|
|
if (!process.env.GRIST_DOCS_MINIO_ACCESS_KEY) {
|
|
this.skip();
|
|
}
|
|
externalStorageCreate = requireStorage(create.getStorageOptions?.('minio')?.create);
|
|
break;
|
|
case 's3':
|
|
if (!process.env.TEST_S3_BUCKET) {
|
|
this.skip();
|
|
}
|
|
externalStorageCreate = requireStorage(create.getStorageOptions?.('s3')?.create);
|
|
break;
|
|
}
|
|
store = new TestStore(tmpDir, workerId, workers, externalStorageCreate);
|
|
});
|
|
|
|
after(async function() {
|
|
await store?.storageManager.testStopOperations();
|
|
await workers?.removeWorker(workerId);
|
|
await cli?.quitAsync();
|
|
});
|
|
|
|
beforeEach(function() {
|
|
sandbox.spy(HostedStorageManager.prototype, 'markAsChanged');
|
|
});
|
|
|
|
afterEach(async function() {
|
|
oldEnv.restore();
|
|
sandbox.restore();
|
|
if (store) {
|
|
await store.end();
|
|
await store.removeAll();
|
|
}
|
|
});
|
|
|
|
async function getRedisChecksum(docId: string): Promise<string> {
|
|
return (await cli.getAsync(`doc-${docId}-checksum`)) || '';
|
|
}
|
|
|
|
async function setRedisChecksum(docId: string, checksum: string): Promise<'OK'> {
|
|
return cli.setAsync(`doc-${docId}-checksum`, checksum);
|
|
}
|
|
|
|
async function dropAllChecksums() {
|
|
// `keys` is a potentially slow, unrecommended operation - but ok in test scenario
|
|
// against a test instance of redis.
|
|
for (const key of await cli.keysAsync('*-checksum')) {
|
|
await cli.delAsync(key);
|
|
}
|
|
}
|
|
|
|
it('can create a fresh empty document', async function() {
|
|
const docId = `create-${uuidv4()}`;
|
|
await workers.assignDocWorker(docId);
|
|
assert.equal(await getRedisChecksum(docId), 'null');
|
|
|
|
// Create an empty document when checksum in redis is 'null'.
|
|
const checksum = await store.run(async () => {
|
|
await store.docManager.fetchDoc(docSession, docId);
|
|
assert(await store.waitForUpdates());
|
|
const checksum = await getRedisChecksum(docId);
|
|
assert.notEqual(checksum, 'null');
|
|
return checksum;
|
|
});
|
|
|
|
// Check what happens when we nobble the expected checksum.
|
|
await setRedisChecksum(docId, 'nobble');
|
|
await store.removeAll();
|
|
|
|
const warnSpy = sandbox.spy(log, 'warn');
|
|
await store.run(async () => {
|
|
await assert.isFulfilled(store.docManager.fetchDoc(docSession, docId));
|
|
assert.isTrue(warnSpy.calledWithMatch('has wrong checksum'), 'a warning should have been logged');
|
|
});
|
|
warnSpy.restore();
|
|
|
|
// Check we get the document back on fresh start if checksum is correct.
|
|
await setRedisChecksum(docId, checksum);
|
|
await store.removeAll();
|
|
await store.run(async () => {
|
|
await store.docManager.fetchDoc(docSession, docId);
|
|
});
|
|
});
|
|
|
|
it('can save modifications', async function() {
|
|
await store.run(async () => {
|
|
await workers.assignDocWorker('Hello');
|
|
await useFixtureDoc('Hello.grist', store.storageManager);
|
|
|
|
await workers.assignDocWorker('Hello2');
|
|
|
|
const doc = await store.docManager.fetchDoc(docSession, 'Hello');
|
|
const doc2 = await store.docManager.fetchDoc(docSession, 'Hello2');
|
|
await doc.docStorage.exec("update Table1 set A = 'magic_word' where id = 1");
|
|
await doc2.docStorage.exec("insert into Table1(id) values(42)");
|
|
return { doc, doc2 };
|
|
});
|
|
|
|
await store.removeAll();
|
|
|
|
await store.run(async () => {
|
|
const doc = await store.docManager.fetchDoc(docSession, 'Hello');
|
|
let result = await doc.docStorage.get("select A from Table1 where id = 1");
|
|
assert.equal(result!.A, 'magic_word');
|
|
const doc2 = await store.docManager.fetchDoc(docSession, 'Hello2');
|
|
result = await doc2.docStorage.get("select id from Table1");
|
|
assert.equal(result!.id, 42);
|
|
});
|
|
});
|
|
|
|
it('can save modifications with interfering backup file', async function() {
|
|
await store.run(async () => {
|
|
// There was a bug where if a corrupt/truncated backup file was created, all future
|
|
// backups would fail. This tickles the condition and makes sure backups now succeed.
|
|
await fse.writeFile(path.join(tmpDir, 'Hello.grist-backup'), 'not a sqlite file');
|
|
|
|
await workers.assignDocWorker('Hello');
|
|
await useFixtureDoc('Hello.grist', store.storageManager);
|
|
|
|
const doc = await store.docManager.fetchDoc(docSession, 'Hello');
|
|
await doc.docStorage.exec("update Table1 set A = 'magic_word2' where id = 1");
|
|
});
|
|
|
|
// S3 should have happened after store.run()
|
|
|
|
await store.removeAll();
|
|
await store.run(async () => {
|
|
const doc = await store.docManager.fetchDoc(docSession, 'Hello');
|
|
const result = await doc.docStorage.get("select A from Table1 where id = 1");
|
|
assert.equal(result!.A, 'magic_word2');
|
|
});
|
|
});
|
|
|
|
it('survives if there is a doc marked dirty that turns out to be clean', async function() {
|
|
await store.run(async () => {
|
|
await workers.assignDocWorker('Hello');
|
|
await useFixtureDoc('Hello.grist', store.storageManager);
|
|
|
|
const doc = await store.docManager.fetchDoc(docSession, 'Hello');
|
|
await doc.docStorage.exec("update Table1 set A = 'magic_word' where id = 1");
|
|
});
|
|
|
|
await store.removeAll();
|
|
|
|
await store.run(async () => {
|
|
const doc = await store.docManager.fetchDoc(docSession, 'Hello');
|
|
const result = await doc.docStorage.get("select A from Table1 where id = 1");
|
|
assert.equal(result!.A, 'magic_word');
|
|
store.docManager.markAsChanged(doc);
|
|
});
|
|
|
|
// The real test is whether this test manages to complete.
|
|
});
|
|
|
|
it('serializes parallel opening of same document', async function() {
|
|
await workers.assignDocWorker('Hello');
|
|
|
|
// put a doc in s3
|
|
await store.run(async () => {
|
|
await useFixtureDoc('Hello.grist', store.storageManager);
|
|
const doc = await store.docManager.fetchDoc(docSession, 'Hello');
|
|
await doc.docStorage.exec("update Table1 set A = 'parallel' where id = 1");
|
|
});
|
|
|
|
// now open it many times in parallel
|
|
await store.removeAll();
|
|
await store.run(async () => {
|
|
const docs = Promise.all([
|
|
store.docManager.fetchDoc(docSession, 'Hello'),
|
|
store.docManager.fetchDoc(docSession, 'Hello'),
|
|
store.docManager.fetchDoc(docSession, 'Hello'),
|
|
store.docManager.fetchDoc(docSession, 'Hello'),
|
|
]);
|
|
await assert.isFulfilled(docs);
|
|
const doc = (await docs)[0];
|
|
const result = await doc.docStorage.get("select A from Table1 where id = 1");
|
|
assert.equal(result!.A, 'parallel');
|
|
});
|
|
|
|
// To be sure we are checking something, let's call prepareLocalDoc directly
|
|
// on storage manager and make sure it fails.
|
|
await store.removeAll();
|
|
await store.run(async () => {
|
|
const preps = Promise.all([
|
|
store.storageManager.prepareLocalDoc('Hello'),
|
|
store.storageManager.prepareLocalDoc('Hello'),
|
|
store.storageManager.prepareLocalDoc('Hello'),
|
|
store.storageManager.prepareLocalDoc('Hello')
|
|
]);
|
|
await assert.isRejected(preps, /in parallel/);
|
|
});
|
|
});
|
|
|
|
it ('can delete a document', async function() {
|
|
const docId = `create-${uuidv4()}`;
|
|
await workers.assignDocWorker(docId);
|
|
|
|
// Create a document
|
|
await store.run(async () => {
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.docStorage.exec("insert into Table1(id) values(42)");
|
|
});
|
|
|
|
const docPath = store.getDocPath(docId);
|
|
const ext = store.storageManager.testGetExternalStorage();
|
|
|
|
// Check that the document exists on filesystem and in external store.
|
|
await store.run(async () => {
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
assert.equal(await fse.pathExists(docPath), true);
|
|
assert.equal(await fse.pathExists(docPath + '-hash-doc'), true);
|
|
await waitForIt(async () => assert.equal(await ext.exists(docId), true), 20000);
|
|
await doc.docStorage.exec("insert into Table1(id) values(43)");
|
|
|
|
// Now delete the document, and check it no longer exists on filesystem or external store.
|
|
await store.docManager.deleteDoc(null, docId, true);
|
|
assert.equal(await fse.pathExists(docPath), false);
|
|
assert.equal(await fse.pathExists(docPath + '-hash-doc'), false);
|
|
assert.equal(await getRedisChecksum(docId), DELETED_TOKEN);
|
|
await waitForIt(async () => assert.equal(await ext.exists(docId), false), 20000);
|
|
});
|
|
|
|
// As far as the underlying storage is concerned it should be
|
|
// possible to recreate a doc with the same id after deletion.
|
|
// This should not happen in Grist, since in order to open a
|
|
// document it must exist in the db - however we'll need to watch
|
|
// out for caching.
|
|
// TODO: it could be worth tweaking fetchDoc so creation is explicit.
|
|
await store.run(async () => {
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.docStorage.exec("insert into Table1(id) values(42)");
|
|
});
|
|
|
|
await store.run(async () => {
|
|
await store.docManager.fetchDoc(docSession, docId);
|
|
assert.equal(await fse.pathExists(docPath), true);
|
|
assert.equal(await fse.pathExists(docPath + '-hash-doc'), true);
|
|
});
|
|
});
|
|
|
|
it('individual document close is orderly', async function() {
|
|
const docId = `create-${uuidv4()}`;
|
|
await workers.assignDocWorker(docId);
|
|
|
|
await store.run(async () => {
|
|
let doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await store.closeDoc(doc);
|
|
const checksum1 = await getRedisChecksum(docId);
|
|
assert.notEqual(checksum1, 'null');
|
|
|
|
doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.docStorage.exec("insert into Table1(id) values(42)");
|
|
|
|
// Add an attachment file with no corresponding metadata. It should be deleted when shutting down.
|
|
await doc.docStorage.exec("insert into _gristsys_Files(id, ident) values(23, 'foo')");
|
|
let files = await doc.docStorage.all("select * from _gristsys_Files");
|
|
assert.isNotEmpty(files);
|
|
|
|
await store.closeDoc(doc);
|
|
const checksum2 = await getRedisChecksum(docId);
|
|
assert.notEqual(checksum1, checksum2);
|
|
|
|
doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.docStorage.exec("insert into Table1(id) values(43)");
|
|
|
|
// Attachment file should have been deleted on previous close.
|
|
files = await doc.docStorage.all("select * from _gristsys_Files");
|
|
assert.isEmpty(files);
|
|
|
|
const asyncClose = store.closeDoc(doc); // this time, don't explicitly wait for closeDoc.
|
|
doc = await store.docManager.fetchDoc(docSession, docId);
|
|
const checksum3 = await getRedisChecksum(docId);
|
|
assert.notEqual(checksum2, checksum3);
|
|
await asyncClose;
|
|
});
|
|
});
|
|
|
|
// Viewing a document should not mark it as changed (unless a document-level migration
|
|
// needed to run).
|
|
it('viewing a document does not generally change it', async function() {
|
|
const docId = `create-${uuidv4()}`;
|
|
await workers.assignDocWorker(docId);
|
|
|
|
await store.run(async () => {
|
|
const markAsChanged: {callCount: number} = store.storageManager.markAsChanged as any;
|
|
|
|
const changesInitial = markAsChanged.callCount;
|
|
let doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.waitForInitialization();
|
|
await store.closeDoc(doc);
|
|
const changesAfterCreation = markAsChanged.callCount;
|
|
assert.isAbove(changesAfterCreation, changesInitial);
|
|
|
|
doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.waitForInitialization();
|
|
await store.closeDoc(doc);
|
|
const changesAfterViewing = markAsChanged.callCount;
|
|
assert.equal(changesAfterViewing, changesAfterCreation);
|
|
});
|
|
});
|
|
|
|
it('can fork documents', async function() {
|
|
const docId = `create-${uuidv4()}`;
|
|
const forkId = `${docId}~fork1`;
|
|
await workers.assignDocWorker(docId);
|
|
await workers.assignDocWorker(forkId);
|
|
|
|
await store.run(async () => {
|
|
await useFixtureDoc('Hello.grist', store.storageManager, `${docId}.grist`);
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.docStorage.exec("update Table1 set A = 'trunk' where id = 1");
|
|
});
|
|
|
|
await store.run(async () => {
|
|
await store.docManager.storageManager.prepareFork(docId, forkId);
|
|
const doc = await store.docManager.fetchDoc(docSession, forkId);
|
|
assert.equal('trunk', (await doc.docStorage.get("select A from Table1 where id = 1"))!.A);
|
|
await doc.docStorage.exec("update Table1 set A = 'fork' where id = 1");
|
|
});
|
|
|
|
await store.removeAll();
|
|
|
|
await store.run(async () => {
|
|
let doc = await store.docManager.fetchDoc(docSession, docId);
|
|
assert.equal('trunk', (await doc.docStorage.get("select A from Table1 where id = 1"))!.A);
|
|
doc = await store.docManager.fetchDoc(docSession, forkId);
|
|
assert.equal('fork', (await doc.docStorage.get("select A from Table1 where id = 1"))!.A);
|
|
});
|
|
|
|
// Check that the trunk can be replaced by a fork
|
|
await store.removeAll();
|
|
await store.run(async () => {
|
|
await store.storageManager.replace(docId, {sourceDocId: forkId});
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
assert.equal('fork', (await doc.docStorage.get("select A from Table1 where id = 1"))!.A);
|
|
});
|
|
});
|
|
|
|
it('can persist a fork with no modifications', async function() {
|
|
const docId = `create-${uuidv4()}`;
|
|
const forkId = `${docId}~fork1`;
|
|
await workers.assignDocWorker(docId);
|
|
await workers.assignDocWorker(forkId);
|
|
|
|
// Create a document.
|
|
await store.run(async () => {
|
|
await useFixtureDoc('Hello.grist', store.storageManager, `${docId}.grist`);
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.docStorage.exec("update Table1 set A = 'trunk' where id = 1");
|
|
});
|
|
|
|
// Create a fork with no modifications.
|
|
await store.run(async () => {
|
|
await store.docManager.storageManager.prepareFork(docId, forkId);
|
|
});
|
|
await store.waitForUpdates();
|
|
await store.removeAll();
|
|
|
|
// Zap local copy of fork.
|
|
await fse.remove(store.getDocPath(docId));
|
|
|
|
// Make sure opening the fork works as expected.
|
|
await store.run(async () => {
|
|
const doc = await store.docManager.fetchDoc(docSession, forkId);
|
|
assert.equal('trunk', (await doc.docStorage.get("select A from Table1 where id = 1"))!.A);
|
|
});
|
|
await store.removeAll();
|
|
});
|
|
|
|
it('can access snapshots', async function() {
|
|
// Keep number of forks less than 5 so pruning doesn't kick in.
|
|
const forks = 4;
|
|
|
|
const docId = `create-${uuidv4()}`;
|
|
const forkId1 = `${docId}~fork1`;
|
|
const forkId2 = `${docId}~fork2`;
|
|
const forkId3 = `${docId}~fork3`;
|
|
await workers.assignDocWorker(docId);
|
|
await workers.assignDocWorker(forkId1);
|
|
await workers.assignDocWorker(forkId2);
|
|
await workers.assignDocWorker(forkId3);
|
|
|
|
const doc = await store.run(async () => {
|
|
await useFixtureDoc('Hello.grist', store.storageManager, `${docId}.grist`);
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.waitForInitialization();
|
|
for (let i = 0; i < forks; i++) {
|
|
await doc.docStorage.exec(`update Table1 set A = 'v${i}' where id = 1`);
|
|
await doc.testKeepOpen();
|
|
await store.waitForUpdates();
|
|
}
|
|
return doc;
|
|
});
|
|
|
|
const {snapshots} = await store.storageManager.getSnapshots(doc.docName);
|
|
assert.isAtLeast(snapshots.length, forks + 1); // May be 1 greater depending on how long
|
|
// it takes to run initial migrations.
|
|
await store.run(async () => {
|
|
for (let i = forks - 1; i >= 0; i--) {
|
|
const snapshot = snapshots.shift()!;
|
|
const forkId = snapshot.docId;
|
|
await workers.assignDocWorker(forkId);
|
|
const doc = await store.docManager.fetchDoc(docSession, forkId);
|
|
assert.equal(`v${i}`, (await doc.docStorage.get("select A from Table1 where id = 1"))!.A);
|
|
}
|
|
});
|
|
});
|
|
|
|
it('can access snapshots with old schema versions', async function() {
|
|
const snapshotId = `World~v=1`;
|
|
await workers.assignDocWorker(snapshotId);
|
|
await store.run(async () => {
|
|
// Pretend we have a snapshot of World-v33.grist and fetch/load it.
|
|
await useFixtureDoc('World-v33.grist', store.storageManager, `${snapshotId}.grist`);
|
|
const doc = await store.docManager.fetchDoc(docSession, snapshotId);
|
|
|
|
// Check that the snapshot isn't broken.
|
|
assert.doesNotThrow(async () => await doc.waitForInitialization());
|
|
|
|
// Check that the snapshot was migrated to the latest schema version.
|
|
assert.equal(
|
|
SCHEMA_VERSION,
|
|
(await doc.docStorage.get("select schemaVersion from _grist_DocInfo where id = 1"))!.schemaVersion
|
|
);
|
|
|
|
// Check that the document is actually a snapshot.
|
|
await assert.isRejected(doc.replace(docSession, {sourceDocId: 'docId'}),
|
|
/Snapshots cannot be replaced/);
|
|
await assert.isRejected(doc.applyUserActions(docSession, [['AddTable', 'NewTable', [{id: 'A'}]]]),
|
|
/pyCall is not available in snapshots/);
|
|
});
|
|
});
|
|
|
|
it('can prune snapshots', async function() {
|
|
const versions = 8;
|
|
|
|
const docId = `create-${uuidv4()}`;
|
|
const doc = await store.run(async () => {
|
|
await useFixtureDoc('Hello.grist', store.storageManager, `${docId}.grist`);
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
for (let i = 0; i < versions; i++) {
|
|
await doc.docStorage.exec(`update Table1 set A = 'v${i}' where id = 1`);
|
|
await doc.testKeepOpen();
|
|
await store.waitForUpdates();
|
|
}
|
|
await store.storageManager.testWaitForPrunes();
|
|
return doc;
|
|
});
|
|
await waitForIt(async () => {
|
|
const {snapshots} = await store.storageManager.getSnapshots(doc.docName);
|
|
// Should be keeping at least five, and then maybe 1 more if the hour changed
|
|
// during the test.
|
|
assert.isAtMost(snapshots.length, 6);
|
|
assert.isAtLeast(snapshots.length, 5);
|
|
}, 20000);
|
|
await waitForIt(async () => {
|
|
// Double check with external store directly.
|
|
const snapshots = await store.storageManager.testGetExternalStorage().versions(doc.docName);
|
|
assert.isAtMost(snapshots.length, 6);
|
|
assert.isAtLeast(snapshots.length, 5);
|
|
}, 20000);
|
|
});
|
|
|
|
for (const wipeLocal of [false, true]) {
|
|
it (`can lose checksums without disruption with${wipeLocal ? '' : 'out'} local file wipe`, async function() {
|
|
const docId = `create-${uuidv4()}`;
|
|
await workers.assignDocWorker(docId);
|
|
|
|
// Create a series of versions of a document, and fetch them sequentially
|
|
// so that they are potentially available as stale values.
|
|
await store.run(async () => {
|
|
await useFixtureDoc('Hello.grist', store.storageManager, `${docId}.grist`);
|
|
await store.docManager.fetchDoc(docSession, docId);
|
|
});
|
|
for (let i = 0; i < 3; i++) {
|
|
await store.removeAll();
|
|
await store.run(async () => {
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
if (i > 0) {
|
|
const prev = await doc.docStorage.get("select A from Table1 where id = 1");
|
|
assert.equal(prev!.A, `magic_word${i - 1}`);
|
|
}
|
|
await doc.docStorage.exec(`update Table1 set A = 'magic_word${i}' where id = 1`);
|
|
});
|
|
}
|
|
|
|
// Wipe all checksums and make sure (1) we don't get any errors and (2) the
|
|
// right version of the document shows up after a while.
|
|
let result: string | undefined;
|
|
await waitForIt(async () => {
|
|
await dropAllChecksums();
|
|
if (wipeLocal) {
|
|
// Optionally wipe all local files.
|
|
await store.removeAll();
|
|
}
|
|
await store.run(async () => {
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
result = (await doc.docStorage.get("select A from Table1 where id = 1"))?.A;
|
|
});
|
|
if (result !== 'magic_word2') {
|
|
throw new Error(`inconsistent result: ${result}`);
|
|
}
|
|
}, 20000);
|
|
assert.equal(result, 'magic_word2');
|
|
});
|
|
}
|
|
|
|
it('can access metadata', async function() {
|
|
const docId = `create-${uuidv4()}`;
|
|
const { tz, h, doc } = await store.run(async () => {
|
|
// Use a doc that's up-to-date on storage migrations, but needs a python schema migration.
|
|
await useFixtureDoc('BlobMigrationV8.grist', store.storageManager, `${docId}.grist`);
|
|
const doc = await store.docManager.fetchDoc(docSession, docId);
|
|
await doc.waitForInitialization();
|
|
const rec = await doc.fetchTable(makeExceptionalDocSession('system'), '_grist_DocInfo');
|
|
const tz = rec.tableData[3].timezone[0];
|
|
const h = (await doc.getRecentStates(makeExceptionalDocSession('system')))[0].h;
|
|
await store.docManager.makeBackup(doc, 'hello');
|
|
return { tz, h, doc };
|
|
});
|
|
const {snapshots} = await store.storageManager.getSnapshots(doc.docName);
|
|
assert.equal(snapshots[0]?.metadata?.label, 'hello');
|
|
// There can be extra snapshots, depending on timing.
|
|
const prevSnapshotWithLabel = snapshots.find((s, idx) => idx > 0 && s.metadata?.label);
|
|
assert.match(String(prevSnapshotWithLabel?.metadata?.label), /migrate-schema/);
|
|
assert.equal(snapshots[0]?.metadata?.tz, String(tz));
|
|
assert.equal(snapshots[0]?.metadata?.h, h);
|
|
});
|
|
});
|
|
}
|
|
});
|
|
|
|
// This is a performance test, to check if the backup settings are plausible.
|
|
describe('backupSqliteDatabase', async function() {
|
|
it('backups are robust to locking', async function() {
|
|
// Takes some time to create large db and play with it.
|
|
this.timeout(20000);
|
|
|
|
const tmpDir = await createTmpDir();
|
|
const src = path.join(tmpDir, "src.db");
|
|
const dest = path.join(tmpDir, "dest.db");
|
|
const db = await SQLiteDB.openDBRaw(src);
|
|
await db.run("create table data(x,y,z)");
|
|
await db.execTransaction(async () => {
|
|
const stmt = await db.prepare("INSERT INTO data VALUES (?,?,?)");
|
|
for (let i = 0; i < 10000; i++) {
|
|
// Silly code to make a long random string to insert.
|
|
// We can make a big db faster this way.
|
|
const str = (new Array(100)).fill(1).map((_: any) => Math.random().toString(2)).join();
|
|
await stmt.run(str, str, str);
|
|
}
|
|
await stmt.finalize();
|
|
});
|
|
const stat = await fse.stat(src);
|
|
assert(stat.size > 150 * 1000 * 1000);
|
|
let done: boolean = false;
|
|
let eventStart: number = 0;
|
|
let eventAction: string = "";
|
|
let eventCount: number = 0;
|
|
function progress(event: BackupEvent) {
|
|
if (event.phase === 'after') {
|
|
// Duration of backup action should never approach the default node-sqlite3 busy_timeout of 1s.
|
|
// If it does, then user actions could be blocked.
|
|
assert.equal(event.action, eventAction);
|
|
assert.isBelow(Date.now() - eventStart, 100);
|
|
eventCount++;
|
|
} else if (event.phase === 'before') {
|
|
eventStart = Date.now();
|
|
eventAction = event.action;
|
|
}
|
|
}
|
|
let backupError: Error|undefined;
|
|
const act = backupSqliteDatabase(src, dest, progress).then(() => done = true)
|
|
.catch((e) => { done = true; backupError = e; });
|
|
assert(!done);
|
|
// Try a series of insertions, to check that db never appears locked to us.
|
|
for (let i = 0; i < 100; i++) {
|
|
await bluebird.delay(10);
|
|
try {
|
|
await db.exec('INSERT INTO data VALUES (1,2,3)');
|
|
} catch (e) {
|
|
log.error('insertion failed, that is bad news, the db was locked for too long');
|
|
throw e;
|
|
}
|
|
}
|
|
assert(!done);
|
|
|
|
// Lock the db up completely for a while.
|
|
await db.exec('PRAGMA locking_mode = EXCLUSIVE');
|
|
await db.exec('BEGIN EXCLUSIVE');
|
|
await bluebird.delay(500);
|
|
await db.exec('COMMIT');
|
|
await db.exec('PRAGMA locking_mode = NORMAL');
|
|
|
|
assert(!done);
|
|
while (!done) {
|
|
// Make sure regular queries don't get in the way of backup completing
|
|
await db.all('select * from data limit 100');
|
|
await bluebird.delay(100);
|
|
}
|
|
await act;
|
|
if (backupError) { throw backupError; }
|
|
|
|
// Make sure we are receiving backup events and checking their timing.
|
|
assert.isAbove(eventCount, 100);
|
|
|
|
// Finally, check the backup looks sane.
|
|
const db2 = await SQLiteDB.openDBRaw(dest);
|
|
assert.lengthOf(await db2.all('select rowid from data'), 10000 + 100);
|
|
});
|
|
});
|