gristlabs_grist-core/app/server/lib/SQLiteDB.ts

579 lines
23 KiB
TypeScript
Raw Normal View History

/**
* SQLiteDB provides a clean Promise-based interface to SQLite along with an organized way to
* specify the initial structure of the database and migrations when this structure changes.
*
* Here's a simple example,
*
* const schemaInfo: SQLiteDB.SchemaInfo = {
* async create(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT)");
* },
* migrations: [
* async function(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT)");
* }
* ],
* }
* const db = await SQLiteDB.openDB("pathToDB", schemaInfo, SQLiteDB.OpenMode.OPEN_CREATE);
*
* Note how the create() function and the first migration are identical here. But they'll diverge
* once we make a change to the schema. E.g. the next change could look like this:
*
* const schemaInfo: SQLiteDB.SchemaInfo = {
* async create(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT, B NUMERIC)");
* },
* migrations: [
* async function(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT)");
* },
* async function(db: SQLiteDB.SQLiteDB) {
* await db.exec("ALTER TABLE Foo ADD COLUMN B NUMERIC");
* }
* ],
* }
* const db = await SQLiteDB.openDB("pathToDB", schemaInfo, SQLiteDB.OpenMode.OPEN_CREATE);
*
* Now a new document will have two columns. A document created with the first version of the code
* will gain a second column when opened with the new code. If a migration happened during open,
* you may examine two properties of the returned db object:
*
* db.migrationBackupPath -- set to the path of the pre-migration backup file.
* db.migrationError -- set to the Error object if the migration failed.
*
* This module uses SQLite's "user_version" pragma to keep track of the version number of a
* migration. It does not require, support, or record backwards migrations, but it will warn of
* inconsistencies that may arise during development. In that case, remember you have a backup
* from each migration.
*
* If you are starting with an existing unversioned DB, the first migration should have code to
* bring such DBs to a common state.
*
* const schemaInfo: SQLiteDB.SchemaInfo = {
* async create(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT)");
* await db.exec("CREATE TABLE Bar (B TEXT)");
* },
* migrations: [
* async function(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE IF NOT EXISTS Foo (A TEXT)");
* await db.exec("CREATE TABLE IF NOT EXISTS Bar (B TEXT)");
* }
* ],
* }
* const db = await SQLiteDB.openDB("pathToDB", schemaInfo, SQLiteDB.OpenMode.OPEN_CREATE);
*
* Once using this module with versioning, future changes would be made by adding one item to the
* "migrations" array, and modifying create() to create correct new documents.
*/
import {ErrorWithCode} from 'app/common/ErrorWithCode';
import {timeFormat} from 'app/common/timeFormat';
import * as docUtils from 'app/server/lib/docUtils';
import * as log from 'app/server/lib/log';
import {fromCallback} from 'app/server/lib/serverUtils';
import * as sqlite3 from '@gristlabs/sqlite3';
import * as assert from 'assert';
import {each} from 'bluebird';
import * as fse from 'fs-extra';
import {RunResult} from 'sqlite3';
import fromPairs = require('lodash/fromPairs');
import isEqual = require('lodash/isEqual');
import noop = require('lodash/noop');
import range = require('lodash/range');
// Describes the result of get() and all() database methods.
export interface ResultRow {
[column: string]: any;
}
// Describes how to create a new DB or migrate an old one. Any changes to the DB must be reflected
// in the 'create' function, and added as new entries in the 'migrations' array. Existing
// 'migration' entries may not be modified; they are used to migrate older DBs.
export interface SchemaInfo {
// Creates a structure for a new DB (i.e. execs CREATE TABLE statements).
readonly create: DBFunc;
// List of functions that perform DB migrations from one version to the next. This array's
// length determines the schema version, which is stored in user_version SQLite property.
//
// The very first migration should normally be identical to the original version of create().
// I.e. initially SchemaInfo should be { create: X, migrations: [X] }, where the two X's
// represent two copies of the same code. Don't go for code reuse here. When the schema is
// modified, you will change it to { create: X2, migrations: [X, Y] }. Keeping the unchanged
// copy of X is important as a reference to see that X + Y produces the same DB as X2.
//
// If you may open DBs created without versioning (e.g. predate use of this module), such DBs
// will go through all migrations including the very first one. In this case, the first
// migration's job is to bring any older DB to the same consistent state.
readonly migrations: ReadonlyArray<DBFunc>;
}
export type DBFunc = (db: SQLiteDB) => Promise<void>;
export enum OpenMode {
OPEN_CREATE, // Open DB or create if doesn't exist (the default mode for sqlite3 module)
OPEN_EXISTING, // Open DB or fail if doesn't exist
2022-02-19 09:46:49 +00:00
OPEN_READONLY, // Open DB in read-only mode or fail if doesn't exist.
CREATE_EXCL, // Create new DB or fail if it already exists.
}
/**
* Callbacks to use if a migration is run, so that backups are made.
*/
export interface MigrationHooks {
beforeMigration?(currentVersion: number, newVersion: number): Promise<void>;
afterMigration?(newVersion: number, success: boolean): Promise<void>;
}
/**
* An interface implemented both by SQLiteDB and DocStorage (by forwarding). Methods
* documented in SQLiteDB.
*/
export interface ISQLiteDB {
exec(sql: string): Promise<void>;
run(sql: string, ...params: any[]): Promise<RunResult>;
get(sql: string, ...params: any[]): Promise<ResultRow|undefined>;
all(sql: string, ...params: any[]): Promise<ResultRow[]>;
prepare(sql: string, ...params: any[]): Promise<sqlite3.Statement>;
execTransaction<T>(callback: () => Promise<T>): Promise<T>;
runAndGetId(sql: string, ...params: any[]): Promise<number>;
requestVacuum(): Promise<boolean>;
}
/**
* Wrapper around sqlite3.Database. This class provides many of the same methods, but promisified.
* In addition, it offers:
*
* SQLiteDB.openDB(): Opens a DB, and initialize or migrate it to correct schema.
* db.execTransaction(cb): Runs a callback in the context of a new DB transaction.
*/
export class SQLiteDB implements ISQLiteDB {
/**
* Opens a database or creates a new one, according to OpenMode enum. The schemaInfo specifies
* how to initialize a new database, and how to migrate an existing one from an older version.
* If the database was migrated, its "migrationBackupPath" property will be set.
*
* If a migration was needed but failed, the DB remains unchanged, and gets opened anyway.
* We report the migration error, and expose it via .migrationError property.
*/
public static async openDB(dbPath: string, schemaInfo: SchemaInfo,
mode: OpenMode = OpenMode.OPEN_CREATE,
hooks: MigrationHooks = {}): Promise<SQLiteDB> {
const db = await SQLiteDB.openDBRaw(dbPath, mode);
const userVersion: number = await db.getMigrationVersion();
// It's possible that userVersion is 0 for a non-empty DB if it was created without this
// module. In that case, we apply migrations starting with the first one.
if (userVersion === 0 && (await isEmpty(db))) {
await db._initNewDB(schemaInfo);
} else if (mode === OpenMode.CREATE_EXCL) {
await db.close();
throw new ErrorWithCode('EEXISTS', `EEXISTS: Database already exists: ${dbPath}`);
} else {
// Don't attempt migrations in OPEN_READONLY mode.
if (mode === OpenMode.OPEN_READONLY) {
const targetVer: number = schemaInfo.migrations.length;
if (userVersion < targetVer) {
db._migrationError = new Error(`SQLiteDB[${dbPath}] needs migration but is readonly`);
}
} else {
try {
db._migrationBackupPath = await db._migrate(userVersion, schemaInfo, hooks);
} catch (err) {
db._migrationError = err;
}
}
await db._reportSchemaDiscrepancies(schemaInfo);
}
return db;
}
/**
* Opens a database or creates a new one according to OpenMode value. Does not check for or do
* any migrations.
*/
public static async openDBRaw(dbPath: string,
mode: OpenMode = OpenMode.OPEN_CREATE): Promise<SQLiteDB> {
const sqliteMode: number =
// tslint:disable-next-line:no-bitwise
(mode === OpenMode.OPEN_READONLY ? sqlite3.OPEN_READONLY : sqlite3.OPEN_READWRITE) |
(mode === OpenMode.OPEN_CREATE || mode === OpenMode.CREATE_EXCL ? sqlite3.OPEN_CREATE : 0);
let _db: sqlite3.Database;
await fromCallback(cb => { _db = new sqlite3.Database(dbPath, sqliteMode, cb); });
limitAttach(_db!, 0); // Outside of VACUUM, we don't allow ATTACH.
if (SQLiteDB._addOpens(dbPath, 1) > 1) {
log.warn("SQLiteDB[%s] avoid opening same DB more than once", dbPath);
}
return new SQLiteDB(_db!, dbPath);
}
/**
* Reads the migration version from the database without any attempts to migrate it.
*/
public static async getMigrationVersion(dbPath: string): Promise<number> {
const db = await SQLiteDB.openDBRaw(dbPath, OpenMode.OPEN_READONLY);
try {
return await db.getMigrationVersion();
} finally {
await db.close();
}
}
// It is a bad idea to open the same database file multiple times, because simultaneous use can
// cause SQLITE_BUSY errors, and artificial delays (default of 1 sec) when there is contention.
// We keep track of open DB paths, and warn if one is opened multiple times.
private static _openPaths: Map<string, number> = new Map();
// Convert the "create" function from schemaInfo into a DBMetadata object that describes the
// tables, columns, and types. This is used for checking if an open database matches the
// schema we expect, including after a migration, and reporting discrepancies.
private static async _getExpectedMetadata(schemaInfo: SchemaInfo): Promise<DBMetadata> {
// We cache the result and associate it with the create function, since it's not that cheap to
// build. To build the metadata, we open an in-memory DB and apply "create" function to it.
// Note that for tiny DBs it takes <10ms.
if (!dbMetadataCache.has(schemaInfo.create)) {
const db = await SQLiteDB.openDB(':memory:', schemaInfo, OpenMode.CREATE_EXCL);
dbMetadataCache.set(schemaInfo.create, await db.collectMetadata());
await db.close();
}
return dbMetadataCache.get(schemaInfo.create)!;
}
// Private helper to keep track of opens for the same path. Returns the number of times this
// path is open, after adding the delta. Use delta of +1 for open, -1 for close.
private static _addOpens(dbPath: string, delta: number): number {
const newCount = (SQLiteDB._openPaths.get(dbPath) || 0) + delta;
if (newCount > 0) {
SQLiteDB._openPaths.set(dbPath, newCount);
} else {
SQLiteDB._openPaths.delete(dbPath);
}
return newCount;
}
private _prevTransaction: Promise<any> = Promise.resolve();
private _inTransaction: boolean = false;
private _migrationBackupPath: string|null = null;
private _migrationError: Error|null = null;
private _needVacuum: boolean = false;
private constructor(private _db: sqlite3.Database, private _dbPath: string) {
// Default database to serialized execution. See https://github.com/mapbox/node-sqlite3/wiki/Control-Flow
// This isn't enough for transactions, which we serialize explicitly.
this._db.serialize();
}
/**
* If a DB was migrated on open, this will be set to the path of the pre-migration backup copy.
* If migration failed, open throws with unchanged DB and no backup file.
*/
public get migrationBackupPath(): string|null { return this._migrationBackupPath; }
/**
* If a needed migration failed, the DB will be opened anyway, with this property set to the
* error. E.g. you may use it like so:
* sdb = await SQLiteDB.openDB(...)
* if (sdb.migrationError) { throw sdb.migrationError; }
*/
public get migrationError(): Error|null { return this._migrationError; }
// The following methods mirror https://github.com/mapbox/node-sqlite3/wiki/API, but return
// Promises. We use fromCallback() rather than use promisify, to get better type-checking.
public exec(sql: string): Promise<void> {
return fromCallback(cb => this._db.exec(sql, cb));
}
public run(sql: string, ...params: any[]): Promise<RunResult> {
return new Promise((resolve, reject) => {
function callback(this: RunResult, err: Error | null) {
if (err) {
reject(err);
} else {
resolve(this);
}
}
this._db.run(sql, ...params, callback);
});
}
public get(sql: string, ...params: any[]): Promise<ResultRow|undefined> {
return fromCallback(cb => this._db.get(sql, ...params, cb));
}
public all(sql: string, ...params: any[]): Promise<ResultRow[]> {
return fromCallback(cb => this._db.all(sql, ...params, cb));
}
public allMarshal(sql: string, ...params: any[]): Promise<Buffer> {
// allMarshal isn't in the typings, because it is our addition to our fork of sqlite3 JS lib.
return fromCallback(cb => (this._db as any).allMarshal(sql, ...params, cb));
}
public prepare(sql: string, ...params: any[]): Promise<sqlite3.Statement> {
let stmt: sqlite3.Statement;
// The original interface is a little strange; we resolve to Statement if prepare() succeeded.
return fromCallback(cb => { stmt = this._db.prepare(sql, ...params, cb); }).then(() => stmt);
}
/**
* VACUUM the DB either immediately or, if in a transaction, after that transaction.
*/
public async requestVacuum(): Promise<boolean> {
if (this._inTransaction) {
this._needVacuum = true;
return false;
}
await this.vacuum();
log.info("SQLiteDB[%s]: DB VACUUMed", this._dbPath);
this._needVacuum = false;
return true;
}
public async vacuum(): Promise<void> {
limitAttach(this._db, 1); // VACUUM implementation uses ATTACH.
try {
await this.exec("VACUUM");
} finally {
limitAttach(this._db, 0); // Outside of VACUUM, we don't allow ATTACH.
}
}
/**
* Run each of the statements in turn. Each statement is either a string, or an array of arguments
* to db.run, e.g. [sqlString, [params...]].
*/
public runEach(...statements: Array<string | [string, any[]]>): Promise<void> {
return each(statements,
async (stmt: any) => {
try {
return await (Array.isArray(stmt) ?
this.run(stmt[0], ...stmt[1]) :
this.exec(stmt)
);
} catch (err) {
log.warn(`SQLiteDB: Failed to run ${stmt}`);
throw err;
}
}
);
}
public close(): Promise<void> {
return fromCallback(cb => this._db.close(cb))
.then(() => { SQLiteDB._addOpens(this._dbPath, -1); });
}
/**
* As for run(), but captures the last_insert_rowid after the statement executes. This
* is sqlite's rowid for the last insert made on this database connection. This method
* is only useful if the sql is actually an INSERT operation, but we don't check this.
*/
public async runAndGetId(sql: string, ...params: any[]): Promise<number> {
const result = await this.run(sql, ...params);
return result.lastID;
}
/**
* Runs callback() in the context of a new DB transaction, committing on success and rolling
* back on error in the callback. The callback may return a promise, which will be waited for.
* The callback is called with no arguments.
*
* This method can be nested. The result is one big merged transaction that will succeed or
* roll back as a single unit.
*/
public async execTransaction<T>(callback: () => Promise<T>): Promise<T> {
if (this._inTransaction) {
return callback();
}
let outerResult;
try {
outerResult = await (this._prevTransaction = this._execTransactionImpl(async () => {
this._inTransaction = true;
let innerResult;
try {
innerResult = await callback();
} finally {
this._inTransaction = false;
}
return innerResult;
}));
} finally {
if (this._needVacuum) {
await this.requestVacuum();
}
}
return outerResult;
}
/**
* Returns the 'user_version' saved in the database that reflects the current DB schema. It is 0
* initially, and we update it to 1 or higher when initializing or migrating the database.
*/
public async getMigrationVersion(): Promise<number> {
const row = await this.get("PRAGMA user_version");
return (row && row.user_version) || 0;
}
/**
* Creates a DBMetadata object mapping DB's table names to column names to column types. Used
* for reporting discrepancies in DB schema, and exposed for tests.
*
* Optionally, a list of table names can be supplied, and metadata will be omitted for any
* tables not named in that list.
*/
public async collectMetadata(names?: string[]): Promise<DBMetadata> {
const tables = await this.all("SELECT name FROM sqlite_master WHERE type='table'");
const metadata: DBMetadata = {};
for (const t of tables) {
if (names && !names.includes(t.name)) { continue; }
const infoRows = await this.all(`PRAGMA table_info(${quoteIdent(t.name)})`);
const columns = fromPairs(infoRows.map(r => [r.name, r.type]));
metadata[t.name] = columns;
}
return metadata;
}
// Implementation of execTransction.
private async _execTransactionImpl<T>(callback: () => Promise<T>): Promise<T> {
// We need to swallow errors, so that one failed transaction doesn't cause the next one to fail.
await this._prevTransaction.catch(noop);
await this.exec("BEGIN");
try {
const value = await callback();
await this.exec("COMMIT");
return value;
} catch (err) {
try {
await this.exec("ROLLBACK");
} catch (rollbackErr) {
log.error("SQLiteDB[%s]: Rollback failed: %s", this._dbPath, rollbackErr);
}
throw err; // Throw the original error from the transaction.
}
}
/**
* Applies schemaInfo.create function to initialize a new DB.
*/
private async _initNewDB(schemaInfo: SchemaInfo): Promise<void> {
await this.execTransaction(async () => {
const targetVer: number = schemaInfo.migrations.length;
await schemaInfo.create(this);
await this.exec(`PRAGMA user_version = ${targetVer}`);
});
}
/**
* Applies migrations to this database according to MigrationInfo. In all cases, checks the
* database schema against MigrationInfo.currentSchema, and warns of discrepancies.
*
* If migration succeeded, it leaves a backup file and returns its path. If no migration was
* needed, returns null. If migration failed, leaves DB unchanged and throws Error.
*/
private async _migrate(actualVer: number, schemaInfo: SchemaInfo,
hooks: MigrationHooks): Promise<string|null> {
const targetVer: number = schemaInfo.migrations.length;
let backupPath: string|null = null;
let success: boolean = false;
if (actualVer > targetVer) {
log.warn("SQLiteDB[%s]: DB is at version %s ahead of target version %s",
this._dbPath, actualVer, targetVer);
} else if (actualVer < targetVer) {
log.info("SQLiteDB[%s]: DB needs migration from version %s to %s",
this._dbPath, actualVer, targetVer);
const versions = range(actualVer, targetVer);
backupPath = await createBackupFile(this._dbPath, actualVer);
await hooks.beforeMigration?.(actualVer, targetVer);
try {
await this.execTransaction(async () => {
for (const versionNum of versions) {
await schemaInfo.migrations[versionNum](this);
}
await this.exec(`PRAGMA user_version = ${targetVer}`);
});
success = true;
// After a migration, reduce the sqlite file size. This must be run outside a transaction.
await this.vacuum();
log.info("SQLiteDB[%s]: DB backed up to %s, migrated to %s",
this._dbPath, backupPath, targetVer);
} catch (err) {
// If the transaction failed, we trust SQLite to have left the DB in unmodified state, so
// we remove the pointless backup.
await fse.remove(backupPath);
backupPath = null;
log.warn("SQLiteDB[%s]: DB migration from %s to %s failed: %s",
this._dbPath, actualVer, targetVer, err);
err.message = `SQLiteDB[${this._dbPath}] migration to ${targetVer} failed: ${err.message}`;
throw err;
} finally {
await hooks.afterMigration?.(targetVer, success);
}
}
return backupPath;
}
private async _reportSchemaDiscrepancies(schemaInfo: SchemaInfo): Promise<void> {
// Regardless of where we started, warn if DB doesn't match expected schema.
const expected = await SQLiteDB._getExpectedMetadata(schemaInfo);
const metadata = await this.collectMetadata(Object.keys(expected));
for (const tname in expected) {
if (expected.hasOwnProperty(tname) && !isEqual(metadata[tname], expected[tname])) {
log.warn("SQLiteDB[%s]: table %s does not match schema: %s != %s",
this._dbPath, tname, JSON.stringify(metadata[tname]), JSON.stringify(expected[tname]));
}
}
}
}
// Every SchemaInfo.create function determines a DB structure. We can get it by initializing a
// dummy DB, and we use it to do sanity checking, in particular after migrations. To avoid
// creating dummy DBs multiple times, the result is cached, keyed by the "create" function itself.
const dbMetadataCache: Map<DBFunc, DBMetadata> = new Map();
interface DBMetadata {
[tableName: string]: {
[colName: string]: string; // Maps column name to SQLite type, e.g. "TEXT".
};
}
// Helper to see if a database is empty.
async function isEmpty(db: SQLiteDB): Promise<boolean> {
return (await db.get("SELECT count(*) as count FROM sqlite_master"))!.count === 0;
}
/**
* Copies filePath to "filePath.YYYY-MM-DD.V0[-N].bak", adding "-N" suffix (starting at "-2") if
* needed to ensure the path is new. Returns the backup path.
*/
async function createBackupFile(filePath: string, versionNum: number): Promise<string> {
const backupPath = await docUtils.createNumberedTemplate(
`${filePath}.${timeFormat('D', new Date())}.V${versionNum}{NUM}.bak`,
docUtils.createExclusive);
await docUtils.copyFile(filePath, backupPath);
return backupPath;
}
/**
* Validate and quote SQL identifiers such as table and column names.
*/
export function quoteIdent(ident: string): string {
assert(/^[\w.]+$/.test(ident), `SQL identifier is not valid: ${ident}`);
return `"${ident}"`;
}
/**
* Limit the number of ATTACHed databases permitted.
*/
export function limitAttach(db: sqlite3.Database, maxAttach: number) {
// Pardon the casts, types are out of date.
const SQLITE_LIMIT_ATTACHED = (sqlite3 as any).LIMIT_ATTACHED;
(db as any).configure('limit', SQLITE_LIMIT_ATTACHED, maxAttach);
}