diff --git a/app/gen-server/lib/TypeORMPatches.ts b/app/gen-server/lib/TypeORMPatches.ts index 12f1bc1a..f67145ff 100644 --- a/app/gen-server/lib/TypeORMPatches.ts +++ b/app/gen-server/lib/TypeORMPatches.ts @@ -13,6 +13,7 @@ // changed during construction of a query. import * as sqlite3 from '@gristlabs/sqlite3'; +import {delay} from 'app/common/delay'; import * as log from 'app/server/lib/log'; import {Mutex, MutexInterface} from 'async-mutex'; import isEqual = require('lodash/isEqual'); @@ -97,27 +98,83 @@ export function applyPatch() { } const queryRunner = this.connection.createQueryRunner(); const runInTransaction = typeof arg1 === "function" ? arg1 : arg2; + const isSqlite = this.connection.driver.options.type === 'sqlite'; try { - await queryRunner.startTransaction(); - const result = await runInTransaction(queryRunner.manager); - await queryRunner.commitTransaction(); - return result; - } catch (err) { - log.debug(`SQLite transaction error [${arg1} ${arg2}] - ${err}`); - try { - // we throw original error even if rollback thrown an error - await queryRunner.rollbackTransaction(); - // tslint: disable-next-line - } catch (rollbackError) { - // tslint: disable-next-line + async function runOrRollback() { + try { + await queryRunner.startTransaction(); + const result = await runInTransaction(queryRunner.manager); + await queryRunner.commitTransaction(); + return result; + } catch (err) { + log.debug(`TypeORM transaction error [${arg1} ${arg2}] - ${err}`); + try { + // we throw original error even if rollback thrown an error + await queryRunner.rollbackTransaction(); + // tslint: disable-next-line + } catch (rollbackError) { + // tslint: disable-next-line + } + throw err; + } + } + if (isSqlite) { + return await callWithRetry(runOrRollback, { + // Transactions may fail immediately if there are connections from + // multiple processes, regardless of busy_timeout setting. Add a + // retry for this kind of failure. This is relevant to tests, which + // use connections from multiple processes, but not to single-process + // instances of Grist, or instances of Grist that use Postgres for the + // home server. + worthRetry: (e) => Boolean(e.message.match(/SQLITE_BUSY/)), + firstDelayMsec: 10, + factor: 1.25, + maxTotalMsec: 3000, + }); + } else { + // When not using SQLite, don't do anything special. + return await runOrRollback(); } - throw err; } finally { await queryRunner.release(); } }; } +/** + * Call an operation, and if it fails with an error that is worth retrying + * (or any error if worthRetry callback is not specified), retry it after + * a delay of firstDelayMsec. Retries are repeated with delays growing by + * the specified factor (or 2.0 if not specified). Stop if maxTotalMsec is + * specified and has passed. + */ +async function callWithRetry(op: () => Promise, options: { + worthRetry?: (err: Error) => boolean, + maxTotalMsec?: number, + firstDelayMsec: number, + factor?: number, +}): Promise { + const startedAt = Date.now(); + let dt = options.firstDelayMsec; + while (true) { // eslint-disable-line no-constant-condition + try { + return await op(); + } catch (e) { + // throw if not worth retrying + if (options.worthRetry && e instanceof Error && !options.worthRetry(e)) { + throw e; + } + // throw if max time has expired + if (options.maxTotalMsec && Date.now() - startedAt > options.maxTotalMsec) { + throw e; + } + // otherwise wait a bit and retry + await delay(dt); + dt *= options.factor ?? 2.0; + } + } +} + /********************** * Patch 2