AsyncPipe; table schemata; migrations; File logging

src/orm/migrations/DatabaseMigrator.ts

@@ -0,0 +1,179 @@
+import {Container, Inject, Injectable} from '../../di'
+import {Migrator} from './Migrator'
+import {DatabaseService} from '../DatabaseService'
+import {FieldType} from '../types'
+import {Migration} from './Migration'
+import {Builder} from '../builder/Builder'
+
+/**
+ * Migrator implementation that tracks applied migrations in a database table.
+ * @todo allow configuring more of this
+ */
+@Injectable()
+export class DatabaseMigrator extends Migrator {
+    @Inject()
+    protected readonly db!: DatabaseService
+
+    @Inject('injector')
+    protected readonly injector!: Container
+
+    /** True if we've initialized the migrator. */
+    protected initialized = false
+
+    public async initialize(): Promise<void> {
+        await super.initialize()
+
+        if ( this.initialized ) {
+            return
+        }
+
+        const schema = this.db.get().schema()
+        if ( !(await schema.hasTable('migrations')) ) {
+            const table = await schema.table('migrations')
+
+            table.primaryKey('id', FieldType.serial).required()
+
+            table.column('identifier')
+                .type(FieldType.varchar)
+                .required()
+
+            table.column('applygroup')
+                .type(FieldType.integer)
+                .required()
+
+            table.column('applydate')
+                .type(FieldType.timestamp)
+                .required()
+
+            await schema.commit(table)
+        }
+
+        this.initialized = true
+    }
+
+    async has(migration: Migration): Promise<boolean> {
+        return this.builder()
+            .connection('default')
+            .select('id')
+            .from('migrations')
+            .where('identifier', '=', migration.identifier)
+            .exists()
+    }
+
+    async markApplied(migrations: Migration | Migration[], applyDate: Date = new Date()): Promise<void> {
+        if ( !Array.isArray(migrations) ) {
+            migrations = [migrations]
+        }
+
+        const applyGroup = await this.getNextGroupIdentifier()
+        const rows = migrations.map(migration => {
+            return {
+                applygroup: applyGroup,
+                applydate: applyDate,
+                identifier: migration.identifier,
+            }
+        })
+
+        await this.builder()
+            .connection('default')
+            .table('migrations')
+            .insert(rows)
+    }
+
+    async unmarkApplied(migrations: Migration | Migration[]): Promise<void> {
+        if ( !Array.isArray(migrations) ) {
+            migrations = [migrations]
+        }
+
+        const identifiers = migrations.map(migration => migration.identifier)
+
+        await this.builder()
+            .connection('default')
+            .table('migrations')
+            .whereIn('identifier', identifiers)
+            .delete()
+    }
+
+    async getLastApplyGroup(): Promise<string[]> {
+        const applyGroup = await this.builder()
+            .connection('default')
+            .select('applygroup')
+            .from('migrations')
+            .get()
+            .max<number>('applygroup')
+
+        return this.builder()
+            .connection('default')
+            .select('identifier')
+            .from('migrations')
+            .where('applygroup', '=', applyGroup)
+            .get()
+            .asyncPipe()
+            .tap(coll => {
+                return coll.pluck<string>('identifier')
+            })
+            .tap(coll => {
+                return coll.all()
+            })
+            .resolve()
+    }
+
+    /**
+     * Helper method to look up the next `applygroup` that should be used.
+     * @protected
+     */
+    protected async getNextGroupIdentifier(): Promise<number> {
+        const current = await this.builder()
+            .connection('default')
+            .select('applygroup')
+            .from('migrations')
+            .get()
+            .max<number>('applygroup')
+
+        return (current ?? 0) + 1
+    }
+
+    /**
+     * Given a list of migration identifiers, filter out those that have been applied.
+     * @override to make this more efficient
+     * @param identifiers
+     * @protected
+     */
+    protected async filterAppliedMigrations(identifiers: string[]): Promise<string[]> {
+        const existing = await this.builder()
+            .connection('default')
+            .select('identifier')
+            .from('migrations')
+            .whereIn('identifier', identifiers)
+            .get()
+            .pluck<string>('identifier')
+
+        return identifiers.filter(id => !existing.includes(id))
+    }
+
+    /**
+     * Given a list of migration identifiers, filter out those that have not been applied.
+     * @override to make this more efficient
+     * @param identifiers
+     * @protected
+     */
+    protected async filterPendingMigrations(identifiers: string[]): Promise<string[]> {
+        const existing = await this.builder()
+            .connection('default')
+            .select('identifier')
+            .from('migrations')
+            .whereIn('identifier', identifiers)
+            .get()
+            .pluck<string>('identifier')
+
+        return existing.all()
+    }
+
+    /**
+     * Get a query builder instance.
+     * @protected
+     */
+    protected builder(): Builder {
+        return this.injector.make<Builder>(Builder)
+    }
+}

src/orm/migrations/Migration.ts

@@ -0,0 +1,39 @@
+import {Injectable} from '../../di'
+import {Awaitable} from '../../util'
+
+/**
+ * Abstract base-class for one-time migrations.
+ */
+@Injectable()
+export abstract class Migration {
+    /** Set by the Migrations unit on load. */
+    protected migrationIdentifier!: string
+
+    /**
+     * Sets the migration identifier.
+     * This is used internally when the Migrations service loads
+     * the migration files to determine the ID from the file-name.
+     * It shouldn't be used externally.
+     * @param name
+     */
+    public setMigrationIdentifier(name: string): void {
+        this.migrationIdentifier = name
+    }
+
+    /**
+     * Get the unique identifier of this migration.
+     */
+    public get identifier(): string {
+        return this.migrationIdentifier
+    }
+
+    /**
+     * Apply the migration.
+     */
+    abstract up(): Awaitable<void>
+
+    /**
+     * Undo the migration.
+     */
+    abstract down(): Awaitable<void>
+}

src/orm/migrations/Migrator.ts

@@ -0,0 +1,295 @@
+import {Container, Inject, Injectable} from '../../di'
+import {Awaitable, collect, ErrorWithContext} from '../../util'
+import {Migration} from './Migration'
+import {Migrations} from '../services/Migrations'
+import {EventBus} from '../../event/EventBus'
+import {ApplyingMigrationEvent} from './events/ApplyingMigrationEvent'
+import {AppliedMigrationEvent} from './events/AppliedMigrationEvent'
+import {RollingBackMigrationEvent} from './events/RollingBackMigrationEvent'
+import {RolledBackMigrationEvent} from './events/RolledBackMigrationEvent'
+import {NothingToMigrateError} from './NothingToMigrateError'
+
+/**
+ * Manages single-run patches/migrations.
+ */
+@Injectable()
+export abstract class Migrator {
+    @Inject()
+    protected readonly migrations!: Migrations
+
+    @Inject()
+    protected readonly bus!: EventBus
+
+    @Inject('injector')
+    protected readonly injector!: Container
+
+    /**
+     * Should resolve true if the given migration has already been applied.
+     * @param migration
+     */
+    public abstract has(migration: Migration): Awaitable<boolean>
+
+    /**
+     * Should mark the given migrations as being applied.
+     *
+     * If a date is specified, then that is the timestamp when the migrations
+     * were applied, otherwise, use `new Date()`.
+     *
+     * @param migrations
+     * @param date
+     */
+    public abstract markApplied(migrations: Migration | Migration[], date?: Date): Awaitable<void>
+
+    /**
+     * Should un-mark the given migrations as being applied.
+     * @param migration
+     */
+    public abstract unmarkApplied(migration: Migration | Migration[]): Awaitable<void>
+
+    /**
+     * Get the identifiers of the last group of migrations that were applied.
+     */
+    public abstract getLastApplyGroup(): Awaitable<string[]>
+
+    /**
+     * Do any initial setup required to get the migrator ready.
+     * This can be overridden by implementation classes to do any necessary setup.
+     */
+    public initialize(): Awaitable<void> {}  // eslint-disable-line @typescript-eslint/no-empty-function
+
+    /**
+     * Apply pending migrations.
+     *
+     * If identifiers are specified, only the pending migrations with those
+     * identifiers are applied. If none are specified, all pending migrations
+     * will be applied.
+     *
+     * @param identifiers
+     */
+    public async migrate(identifiers?: string[]): Promise<void> {
+        await this.initialize()
+
+        if ( !identifiers ) {
+            identifiers = this.getAllMigrationIdentifiers()
+        }
+
+        identifiers = (await this.filterAppliedMigrations(identifiers)).sort()
+        if ( !identifiers.length ) {
+            throw new NothingToMigrateError()
+        }
+
+        const migrations = collect(identifiers)
+            .map(id => {
+                const migration = this.migrations.get(id)
+
+                if ( !migration ) {
+                    throw new ErrorWithContext(`Unable to find migration with identifier: ${id}`, {
+                        identifier: id,
+                    })
+                }
+
+                return migration
+            })
+
+        await migrations.promiseMap(migration => {
+            return this.apply(migration)
+        })
+
+        await this.markApplied(migrations.all())
+    }
+
+    /**
+     * Rollback applied migrations.
+     *
+     * If specified, only applied migrations with the given identifiers will
+     * be rolled back. If not specified, then the last "batch" of applied
+     * migrations will be rolled back.
+     *
+     * @param identifiers
+     */
+    public async rollback(identifiers?: string[]): Promise<void> {
+        await this.initialize()
+
+        if ( !identifiers ) {
+            identifiers = await this.getLastApplyGroup()
+        }
+
+        identifiers = (await this.filterPendingMigrations(identifiers)).sort()
+        if ( !identifiers.length ) {
+            throw new NothingToMigrateError()
+        }
+
+        const migrations = collect(identifiers)
+            .map(id => {
+                const migration = this.migrations.get(id)
+
+                if ( !migration ) {
+                    throw new ErrorWithContext(`Unable to find migration with identifier: ${id}`, {
+                        identifier: id,
+                    })
+                }
+
+                return migration
+            })
+
+        await migrations.promiseMap(migration => {
+            return this.undo(migration)
+        })
+
+        await this.unmarkApplied(migrations.all())
+    }
+
+    /**
+     * Apply a single migration.
+     * @param migration
+     */
+    public async apply(migration: Migration): Promise<void> {
+        await this.initialize()
+
+        await this.applying(migration)
+
+        await migration.up()
+
+        await this.applied(migration)
+    }
+
+    /**
+     * Rollback a single migration.
+     * @param migration
+     */
+    public async undo(migration: Migration): Promise<void> {
+        await this.initialize()
+
+        await this.rollingBack(migration)
+
+        await migration.down()
+
+        await this.rolledBack(migration)
+    }
+
+    /**
+     * Get all registered migrations, by their string-form identifiers.
+     * @protected
+     */
+    protected getAllMigrationIdentifiers(): string[] {
+        return collect<string>(this.migrations.namespaces())
+            .map(nsp => {
+                return this.migrations.all(nsp)
+                    .map(id => `${nsp}:${id}`)
+            })
+            .tap(coll => {
+                // non-namespaced migrations
+                coll.push(this.migrations.all())
+                return coll
+            })
+            .reduce((current, item) => {
+                return current.concat(item)
+            }, [])
+    }
+
+    /**
+     * Given a list of migration identifiers, filter out those that have been applied.
+     * @param identifiers
+     * @protected
+     */
+    protected async filterAppliedMigrations(identifiers: string[]): Promise<string[]> {
+        return collect(identifiers)
+            .partialMap(identifier => {
+                const migration = this.migrations.get(identifier)
+                if ( migration ) {
+                    return {
+                        identifier,
+                        migration,
+                    }
+                }
+            })
+            .asyncPipe()
+            .tap(coll => {
+                return coll.promiseMap(async group => {
+                    return {
+                        ...group,
+                        has: await this.has(group.migration),
+                    }
+                })
+            })
+            .tap(coll => {
+                return coll.filter(group => !group.has)
+                    .pluck<string>('identifier')
+                    .all()
+            })
+            .resolve()
+    }
+
+    /**
+     * Given a list of migration identifiers, filter out those that have not been applied.
+     * @param identifiers
+     * @protected
+     */
+    protected async filterPendingMigrations(identifiers: string[]): Promise<string[]> {
+        return collect(identifiers)
+            .partialMap(identifier => {
+                const migration = this.migrations.get(identifier)
+                if ( migration ) {
+                    return {
+                        identifier,
+                        migration,
+                    }
+                }
+            })
+            .asyncPipe()
+            .tap(coll => {
+                return coll.promiseMap(async group => {
+                    return {
+                        ...group,
+                        has: await this.has(group.migration),
+                    }
+                })
+            })
+            .tap(coll => {
+                return coll.filter(group => group.has)
+                    .pluck<string>('identifier')
+                    .all()
+            })
+            .resolve()
+    }
+
+    /**
+     * Fire the ApplyingMigrationEvent.
+     * @param migration
+     * @protected
+     */
+    protected async applying(migration: Migration): Promise<void> {
+        const event = <ApplyingMigrationEvent> this.injector.make(ApplyingMigrationEvent, migration)
+        await this.bus.dispatch(event)
+    }
+
+    /**
+     * Fire the AppliedMigrationEvent.
+     * @param migration
+     * @protected
+     */
+    protected async applied(migration: Migration): Promise<void> {
+        const event = <AppliedMigrationEvent> this.injector.make(AppliedMigrationEvent, migration)
+        await this.bus.dispatch(event)
+    }
+
+    /**
+     * Fire the RollingBackMigrationEvent.
+     * @param migration
+     * @protected
+     */
+    protected async rollingBack(migration: Migration): Promise<void> {
+        const event = <RollingBackMigrationEvent> this.injector.make(RollingBackMigrationEvent, migration)
+        await this.bus.dispatch(event)
+    }
+
+    /**
+     * Fire the RolledBackMigrationEvent.
+     * @param migration
+     * @protected
+     */
+    protected async rolledBack(migration: Migration): Promise<void> {
+        const event = <RolledBackMigrationEvent> this.injector.make(RolledBackMigrationEvent, migration)
+        await this.bus.dispatch(event)
+    }
+}

src/orm/migrations/MigratorFactory.ts

@@ -0,0 +1,81 @@
+import {
+    AbstractFactory,
+    DependencyRequirement,
+    PropertyDependency,
+    isInstantiable,
+    DEPENDENCY_KEYS_METADATA_KEY,
+    DEPENDENCY_KEYS_PROPERTY_METADATA_KEY, Instantiable, Injectable, Inject,
+} from '../../di'
+import {Collection, ErrorWithContext} from '../../util'
+import {Logging} from '../../service/Logging'
+import {Config} from '../../service/Config'
+import {Migrator} from './Migrator'
+import {DatabaseMigrator} from './DatabaseMigrator'
+
+/**
+ * A dependency injection factory that matches the abstract Migrator class
+ * and produces an instance of the configured session driver implementation.
+ */
+@Injectable()
+export class MigratorFactory extends AbstractFactory<Migrator> {
+    @Inject()
+    protected readonly logging!: Logging
+
+    @Inject()
+    protected readonly config!: Config
+
+    constructor() {
+        super({})
+    }
+
+    produce(): Migrator {
+        return new (this.getMigratorClass())()
+    }
+
+    match(something: unknown): boolean {
+        return something === Migrator
+    }
+
+    getDependencyKeys(): Collection<DependencyRequirement> {
+        const meta = Reflect.getMetadata(DEPENDENCY_KEYS_METADATA_KEY, this.getMigratorClass())
+        if ( meta ) {
+            return meta
+        }
+
+        return new Collection<DependencyRequirement>()
+    }
+
+    getInjectedProperties(): Collection<PropertyDependency> {
+        const meta = new Collection<PropertyDependency>()
+        let currentToken = this.getMigratorClass()
+
+        do {
+            const loadedMeta = Reflect.getMetadata(DEPENDENCY_KEYS_PROPERTY_METADATA_KEY, currentToken)
+            if ( loadedMeta ) {
+                meta.concat(loadedMeta)
+            }
+            currentToken = Object.getPrototypeOf(currentToken)
+        } while (Object.getPrototypeOf(currentToken) !== Function.prototype && Object.getPrototypeOf(currentToken) !== Object.prototype)
+
+        return meta
+    }
+
+    /**
+     * Return the instantiable class of the configured migrator backend.
+     * @protected
+     * @return Instantiable<Migrator>
+     */
+    protected getMigratorClass(): Instantiable<Migrator> {
+        const MigratorClass = this.config.get('database.migrations.driver', DatabaseMigrator)
+
+        if ( !isInstantiable(MigratorClass) || !(MigratorClass.prototype instanceof Migrator) ) {
+            const e = new ErrorWithContext('Provided migration driver class does not extend from @extollo/lib.Migrator')
+            e.context = {
+                configKey: 'database.migrations.driver',
+                class: MigratorClass.toString(),
+            }
+        }
+
+        return MigratorClass
+    }
+}

src/orm/migrations/NothingToMigrateError.ts

@@ -0,0 +1,14 @@
+import {ErrorWithContext} from '../../util'
+
+/**
+ * Error thrown when the migrator is run, but no migrations need
+ * to be applied/rolled-back.
+ */
+export class NothingToMigrateError extends ErrorWithContext {
+    constructor(
+        message = 'There is nothing to migrate',
+        context?: {[key: string]: any},
+    ) {
+        super(message, context)
+    }
+}

src/orm/migrations/events/AppliedMigrationEvent.ts

@@ -0,0 +1,8 @@
+import {Injectable} from '../../../di'
+import {MigrationEvent} from './MigrationEvent'
+
+/**
+ * Event fired after a migration is applied.
+ */
+@Injectable()
+export class AppliedMigrationEvent extends MigrationEvent {}

src/orm/migrations/events/ApplyingMigrationEvent.ts

@@ -0,0 +1,8 @@
+import {Injectable} from '../../../di'
+import {MigrationEvent} from './MigrationEvent'
+
+/**
+ * Event fired before a migration is applied.
+ */
+@Injectable()
+export class ApplyingMigrationEvent extends MigrationEvent {}

src/orm/migrations/events/MigrationEvent.ts

@@ -0,0 +1,49 @@
+import {Event} from '../../../event/Event'
+import {Migration} from '../Migration'
+import {Inject, Injectable} from '../../../di'
+import {Migrations} from '../../services/Migrations'
+import {ErrorWithContext} from '../../../util'
+
+/**
+ * Generic base-class for migration-related events.
+ */
+@Injectable()
+export abstract class MigrationEvent extends Event {
+    @Inject()
+    protected readonly migrations!: Migrations
+
+    /** The migration relevant to this event. */
+    private internalMigration: Migration
+
+    /**
+     * Get the relevant migration.
+     */
+    public get migration(): Migration {
+        return this.internalMigration
+    }
+
+    constructor(
+        migration: Migration,
+    ) {
+        super()
+        this.internalMigration = migration
+    }
+
+    dehydrate(): {identifier: string} {
+        return {
+            identifier: this.migration.identifier,
+        }
+    }
+
+    rehydrate(state: {identifier: string}): void {
+        const migration = this.migrations.get(state.identifier)
+
+        if ( !migration ) {
+            throw new ErrorWithContext(`Unable to find migration with identifier: ${state.identifier}`, {
+                identifier: state.identifier,
+            })
+        }
+
+        this.internalMigration = migration
+    }
+}

src/orm/migrations/events/RolledBackMigrationEvent.ts

@@ -0,0 +1,8 @@
+import {Injectable} from '../../../di'
+import {MigrationEvent} from './MigrationEvent'
+
+/**
+ * Event fired after a migration has been rolled-back.
+ */
+@Injectable()
+export class RolledBackMigrationEvent extends MigrationEvent {}

src/orm/migrations/events/RollingBackMigrationEvent.ts

@@ -0,0 +1,8 @@
+import {Injectable} from '../../../di'
+import {MigrationEvent} from './MigrationEvent'
+
+/**
+ * Event fired before a migration is rolled back.
+ */
+@Injectable()
+export class RollingBackMigrationEvent extends MigrationEvent {}