Browse Source

AsyncPipe; table schemata; migrations; File logging

orm-types
Garrett Mills 11 months ago
parent
commit
fcce28081b
Signed by: garrettmills GPG Key ID: D2BF5FBA8298F246
  1. 7
      src/lifecycle/Application.ts
  2. 39
      src/migrations/2021-07-23T19:44:00.000Z_CreateSessionsTable.migration.ts
  3. 47
      src/migrations/2021-07-24T10:31:00.000Z_CreateUsersTable.migration.ts
  4. 2
      src/orm/DatabaseService.ts
  5. 29
      src/orm/builder/AbstractBuilder.ts
  6. 3
      src/orm/builder/Builder.ts
  7. 15
      src/orm/connection/Connection.ts
  8. 6
      src/orm/connection/PostgresConnection.ts
  9. 292
      src/orm/dialect/PostgreSQLDialect.ts
  10. 132
      src/orm/dialect/SQLDialect.ts
  11. 48
      src/orm/directive/CreateMigrationDirective.ts
  12. 117
      src/orm/directive/MigrateDirective.ts
  13. 102
      src/orm/directive/RollbackDirective.ts
  14. 25
      src/orm/index.ts
  15. 179
      src/orm/migrations/DatabaseMigrator.ts
  16. 39
      src/orm/migrations/Migration.ts
  17. 295
      src/orm/migrations/Migrator.ts
  18. 81
      src/orm/migrations/MigratorFactory.ts
  19. 14
      src/orm/migrations/NothingToMigrateError.ts
  20. 8
      src/orm/migrations/events/AppliedMigrationEvent.ts
  21. 8
      src/orm/migrations/events/ApplyingMigrationEvent.ts
  22. 49
      src/orm/migrations/events/MigrationEvent.ts
  23. 8
      src/orm/migrations/events/RolledBackMigrationEvent.ts
  24. 8
      src/orm/migrations/events/RollingBackMigrationEvent.ts
  25. 345
      src/orm/schema/PostgresSchema.ts
  26. 37
      src/orm/schema/Schema.ts
  27. 677
      src/orm/schema/TableBuilder.ts
  28. 15
      src/orm/services/Database.ts
  29. 98
      src/orm/services/Migrations.ts
  30. 40
      src/orm/template/migration.ts
  31. 53
      src/orm/types.ts
  32. 51
      src/service/Canonical.ts
  33. 6
      src/service/CanonicalInstantiable.ts
  34. 35
      src/util/collection/AsyncCollection.ts
  35. 35
      src/util/collection/Collection.ts
  36. 1
      src/util/index.ts
  37. 44
      src/util/logging/FileLogger.ts
  38. 3
      src/util/logging/Logger.ts
  39. 148
      src/util/support/Pipe.ts
  40. 22
      src/util/support/data.ts
  41. 14
      src/util/support/string.ts
  42. 6
      src/util/support/types.ts

7
src/lifecycle/Application.ts

@ -15,6 +15,7 @@ import {RunLevelErrorHandler} from './RunLevelErrorHandler'
import {Unit, UnitStatus} from './Unit'
import * as dotenv from 'dotenv'
import {CacheFactory} from '../support/cache/CacheFactory'
import {FileLogger} from '../util/logging/FileLogger'
/**
* Helper function that resolves and infers environment variable values.
@ -225,6 +226,12 @@ export class Application extends Container {
const logging: Logging = this.make<Logging>(Logging)
logging.registerLogger(standard)
if ( this.env('EXTOLLO_LOGGING_ENABLE_FILE') ) {
const file: FileLogger = this.make<FileLogger>(FileLogger)
logging.registerLogger(file)
}
logging.verbose('Attempting to load logging level from the environment...')
const envLevel = this.env('EXTOLLO_LOGGING_LEVEL')

39
src/migrations/2021-07-23T19:44:00.000Z_CreateSessionsTable.migration.ts

@ -0,0 +1,39 @@
import {Inject, Injectable} from '../di'
import {ConstraintType, DatabaseService, FieldType, Migration, Schema} from '../orm'
/**
* Migration that creates the sessions table used by the ORMSession backend.
*/
@Injectable()
export default class CreateSessionsTableMigration extends Migration {
@Inject()
protected readonly db!: DatabaseService
async up(): Promise<void> {
const schema: Schema = this.db.get().schema()
const table = await schema.table('sessions')
table.primaryKey('session_uuid', FieldType.varchar)
.required()
table.column('session_data')
.type(FieldType.json)
.required()
.default('{}')
table.constraint('session_uuid_ck')
.type(ConstraintType.Check)
.expression('LENGTH(session_uuid) > 0')
await schema.commit(table)
}
async down(): Promise<void> {
const schema: Schema = this.db.get().schema()
const table = await schema.table('sessions')
table.dropIfExists()
await schema.commit(table)
}
}

47
src/migrations/2021-07-24T10:31:00.000Z_CreateUsersTable.migration.ts

@ -0,0 +1,47 @@
import {Inject, Injectable} from '../di'
import {DatabaseService, FieldType, Migration, Schema} from '../orm'
/**
* Migration that creates the users table used by @extollo/lib.auth.
*/
@Injectable()
export default class CreateUsersTableMigration extends Migration {
@Inject()
protected readonly db!: DatabaseService
async up(): Promise<void> {
const schema: Schema = this.db.get().schema()
const table = await schema.table('users')
table.primaryKey('user_id')
.required()
table.column('first_name')
.type(FieldType.varchar)
.required()
table.column('last_name')
.type(FieldType.varchar)
.required()
table.column('password_hash')
.type(FieldType.text)
.nullable()
table.column('username')
.type(FieldType.varchar)
.required()
.unique()
await schema.commit(table)
}
async down(): Promise<void> {
const schema: Schema = this.db.get().schema()
const table = await schema.table('users')
table.dropIfExists()
await schema.commit(table)
}
}

2
src/orm/DatabaseService.ts

@ -41,7 +41,7 @@ export class DatabaseService extends AppClass {
* Get a connection instance by its name. Throws if none exists.
* @param name
*/
get(name: string): Connection {
get(name = 'default'): Connection {
if ( !this.has(name) ) {
throw new ErrorWithContext(`No such connection is registered: ${name}`)
}

29
src/orm/builder/AbstractBuilder.ts

@ -1,4 +1,4 @@
import {Inject} from '../../di'
import {Inject, Injectable} from '../../di'
import {DatabaseService} from '../DatabaseService'
import {
Constraint, ConstraintConnectionOperator,
@ -9,7 +9,7 @@ import {
SpecifiedField,
} from '../types'
import {Connection} from '../connection/Connection'
import {deepCopy, ErrorWithContext} from '../../util'
import {deepCopy, ErrorWithContext, Maybe} from '../../util'
import {EscapeValue, QuerySafeValue, raw} from '../dialect/SQLDialect'
import {ResultCollection} from './result/ResultCollection'
import {AbstractResultIterable} from './result/AbstractResultIterable'
@ -24,6 +24,7 @@ export type ConstraintGroupClosure<T> = (group: AbstractBuilder<T>) => any
* A base class that facilitates building database queries using a fluent interface.
* This can be specialized by child-classes to yield query results of the given type `T`.
*/
@Injectable()
export abstract class AbstractBuilder<T> extends AppClass {
@Inject()
protected readonly databaseService!: DatabaseService
@ -55,6 +56,9 @@ export abstract class AbstractBuilder<T> extends AppClass {
/** The connection on which the query should be executed. */
protected registeredConnection?: Connection
/** Raw SQL to use instead. Overrides builder methods. */
protected rawSql?: string
/**
* Create a new, empty, instance of the current builder.
*/
@ -80,6 +84,7 @@ export abstract class AbstractBuilder<T> extends AppClass {
bldr.registeredGroupings = deepCopy(this.registeredGroupings)
bldr.registeredOrders = deepCopy(this.registeredOrders)
bldr.registeredConnection = this.registeredConnection
bldr.rawSql = this.rawSql
return bldr
}
@ -115,6 +120,11 @@ export abstract class AbstractBuilder<T> extends AppClass {
return deepCopy(this.registeredOrders)
}
/** Get the raw SQL overriding the builder methods, if it exists. */
public get appliedRawSql(): Maybe<string> {
return this.rawSql
}
/** Get the source table for this query. */
public get querySource(): QuerySource | undefined {
if ( this.source ) {
@ -555,6 +565,21 @@ export abstract class AbstractBuilder<T> extends AppClass {
return Boolean(result.rows.first())
}
/**
* Set the query manually. Overrides any builder methods.
* @example
* ```ts
* (new Builder())
* .raw('SELECT NOW() AS example_column')
* .get()
* ```
* @param sql
*/
raw(sql: string): this {
this.rawSql = sql
return this
}
/**
* Adds a constraint to this query. This is used internally by the various `where`, `whereIn`, `orWhereNot`, &c.
* @param preop

3
src/orm/builder/Builder.ts

@ -1,5 +1,5 @@
import {ErrorWithContext} from '../../util'
import {Container} from '../../di'
import {Container, Injectable} from '../../di'
import {ResultIterable} from './result/ResultIterable'
import {QueryRow} from '../types'
import {AbstractBuilder} from './AbstractBuilder'
@ -8,6 +8,7 @@ import {AbstractResultIterable} from './result/AbstractResultIterable'
/**
* Implementation of the abstract builder class that returns simple QueryRow objects.
*/
@Injectable()
export class Builder extends AbstractBuilder<QueryRow> {
public getNewInstance(): AbstractBuilder<QueryRow> {
return Container.getContainer().make<Builder>(Builder)

15
src/orm/connection/Connection.ts

@ -5,6 +5,7 @@ import {AppClass} from '../../lifecycle/AppClass'
import {Inject, Injectable} from '../../di'
import {EventBus} from '../../event/EventBus'
import {QueryExecutedEvent} from './event/QueryExecutedEvent'
import {Schema} from '../schema/Schema'
/**
* Error thrown when a connection is used before it is ready.
@ -61,15 +62,11 @@ export abstract class Connection extends AppClass {
*/
public abstract close(): Promise<void>
// public abstract databases(): Promise<Collection<Database>>
// public abstract database(name: string): Promise<Database | undefined>
// public abstract database_as_schema(name: string): Promise<Database>
// public abstract tables(database_name: string): Promise<Collection<Table>>
// public abstract table(database_name: string, table_name: string): Promise<Table | undefined>
/**
* Get a Schema on this connection.
* @param name
*/
public abstract schema(name?: string): Schema
/**
* Fire a QueryExecutedEvent for the given query string.

6
src/orm/connection/PostgresConnection.ts

@ -6,6 +6,8 @@ import {collect} from '../../util'
import {SQLDialect} from '../dialect/SQLDialect'
import {PostgreSQLDialect} from '../dialect/PostgreSQLDialect'
import {Logging} from '../../service/Logging'
import {Schema} from '../schema/Schema'
import {PostgresSchema} from '../schema/PostgresSchema'
/**
* Type interface representing the config for a PostgreSQL connection.
@ -67,4 +69,8 @@ export class PostgresConnection extends Connection {
})
}
}
public schema(name?: string): Schema {
return new PostgresSchema(this, name)
}
}

292
src/orm/dialect/PostgreSQLDialect.ts

@ -1,9 +1,13 @@
import {EscapeValue, QuerySafeValue, raw, SQLDialect} from './SQLDialect'
import {Constraint, isConstraintGroup, isConstraintItem, SpecifiedField} from '../types'
import {Constraint, inverseFieldType, isConstraintGroup, isConstraintItem, SpecifiedField} from '../types'
import {AbstractBuilder} from '../builder/AbstractBuilder'
import {ColumnBuilder, ConstraintBuilder, ConstraintType, IndexBuilder, TableBuilder} from '../schema/TableBuilder'
import {ErrorWithContext, Maybe} from '../../util'
/**
* An implementation of the SQLDialect specific to PostgreSQL.
* @todo joins
* @todo sub-selects
*/
export class PostgreSQLDialect extends SQLDialect {
@ -29,7 +33,7 @@ export class PostgreSQLDialect extends SQLDialect {
`${pad(value.getSeconds())}`,
]
return new QuerySafeValue(value, `${y}-${m}-${d} ${h}:${i}:${s}`)
return new QuerySafeValue(value, `'${y}-${m}-${d} ${h}:${i}:${s}'`)
} else if ( !isNaN(Number(value)) ) {
return new QuerySafeValue(value, String(Number(value)))
} else if ( value === null || typeof value === 'undefined' ) {
@ -55,7 +59,7 @@ export class PostgreSQLDialect extends SQLDialect {
'FROM (',
...query.split('\n').map(x => ` ${x}`),
') AS extollo_target_query',
`OFFSET ${start} LIMIT ${(end - start) + 1}`,
`OFFSET ${start} LIMIT ${(end - start) + 1}`, // FIXME - the +1 is only needed when start === end
].join('\n')
}
@ -85,6 +89,11 @@ export class PostgreSQLDialect extends SQLDialect {
}
public renderSelect(builder: AbstractBuilder<any>): string {
const rawSql = builder.appliedRawSql
if ( rawSql ) {
return rawSql
}
const indent = (item: string, level = 1) => Array(level + 1).fill('')
.join(' ') + item
const queryLines = [
@ -147,6 +156,11 @@ export class PostgreSQLDialect extends SQLDialect {
// TODO support FROM, RETURNING
public renderUpdate(builder: AbstractBuilder<any>, data: {[key: string]: EscapeValue}): string {
const rawSql = builder.appliedRawSql
if ( rawSql ) {
return rawSql
}
const queryLines: string[] = []
// Add table source
@ -171,6 +185,15 @@ export class PostgreSQLDialect extends SQLDialect {
}
public renderExistential(builder: AbstractBuilder<any>): string {
const rawSql = builder.appliedRawSql
if ( rawSql ) {
return `
SELECT EXISTS(
${rawSql}
)
`
}
const query = builder.clone()
.clearFields()
.field(raw('TRUE'))
@ -181,6 +204,11 @@ export class PostgreSQLDialect extends SQLDialect {
// FIXME: subquery support here and with select
public renderInsert(builder: AbstractBuilder<any>, data: {[key: string]: EscapeValue}|{[key: string]: EscapeValue}[] = []): string {
const rawSql = builder.appliedRawSql
if ( rawSql ) {
return rawSql
}
const indent = (item: string, level = 1) => Array(level + 1).fill('')
.join(' ') + item
const queryLines: string[] = []
@ -188,6 +216,11 @@ export class PostgreSQLDialect extends SQLDialect {
if ( !Array.isArray(data) ) {
data = [data]
}
if ( data.length < 1 ) {
return ''
}
const columns = Object.keys(data[0])
// Add table source
@ -227,6 +260,11 @@ export class PostgreSQLDialect extends SQLDialect {
}
public renderDelete(builder: AbstractBuilder<any>): string {
const rawSql = builder.appliedRawSql
if ( rawSql ) {
return rawSql
}
const indent = (item: string, level = 1) => Array(level + 1).fill('')
.join(' ') + item
const queryLines: string[] = []
@ -270,6 +308,11 @@ export class PostgreSQLDialect extends SQLDialect {
if ( isConstraintGroup(constraint) ) {
statements.push(`${indent}${statements.length < 1 ? '' : constraint.preop + ' '}(\n${constraintsToSql(constraint.items, level + 1)}\n${indent})`)
} else if ( isConstraintItem(constraint) ) {
if ( Array.isArray(constraint.operand) && !constraint.operand.length ) {
statements.push(`${indent}1 = 0 -- ${constraint.field} ${constraint.operator} empty set`)
continue
}
const field: string = constraint.field.split('.').map(x => `"${x}"`)
.join('.')
statements.push(`${indent}${statements.length < 1 ? '' : constraint.preop + ' '}${field} ${constraint.operator} ${this.escape(constraint.operand).value}`)
@ -294,4 +337,247 @@ export class PostgreSQLDialect extends SQLDialect {
return ['SET', ...sets].join('\n')
}
public renderCreateTable(builder: TableBuilder): string {
const cols = this.renderTableColumns(builder).map(x => ` ${x}`)
const builderConstraints = builder.getConstraints()
const constraints: string[] = []
for ( const constraintName in builderConstraints ) {
if ( !Object.prototype.hasOwnProperty.call(builderConstraints, constraintName) ) {
continue
}
const constraintBuilder = builderConstraints[constraintName]
const constraintDefinition = this.renderConstraintDefinition(constraintBuilder)
if ( constraintDefinition ) {
constraints.push(` CONSTRAINT ${constraintDefinition}`)
}
}
const parts = [
`CREATE TABLE ${builder.isSkippedIfExisting() ? 'IF NOT EXISTS ' : ''}${builder.name} (`,
[
...cols,
...constraints,
].join(',\n'),
`)`,
]
return parts.join('\n')
}
public renderTableColumns(builder: TableBuilder): string[] {
const defined = builder.getColumns()
const rendered: string[] = []
for ( const columnName in defined ) {
if ( !Object.prototype.hasOwnProperty.call(defined, columnName) ) {
continue
}
const columnBuilder = defined[columnName]
rendered.push(this.renderColumnDefinition(columnBuilder))
}
return rendered
}
/**
* Given a constraint schema-builder, render the constraint definition.
* @param builder
* @protected
*/
protected renderConstraintDefinition(builder: ConstraintBuilder): Maybe<string> {
const constraintType = builder.getType()
if ( constraintType === ConstraintType.Unique ) {
const fields = builder.getFields()
.map(x => `"${x}"`)
.join(',')
return `${builder.name} UNIQUE(${fields})`
} else if ( constraintType === ConstraintType.Check ) {
const expression = builder.getExpression()
if ( !expression ) {
throw new ErrorWithContext('Cannot create check constraint without expression.', {
constraintName: builder.name,
tableName: builder.parent.name,
})
}
return `${builder.name} CHECK(${expression})`
}
}
/**
* Given a column-builder, render the SQL-definition as used in
* CREATE TABLE and ALTER TABLE statements.
* @fixme Type `serial` only exists on CREATE TABLE... queries
* @param builder
* @protected
*/
protected renderColumnDefinition(builder: ColumnBuilder): string {
const type = builder.getType()
if ( !type ) {
throw new ErrorWithContext(`Missing field type for column: ${builder.name}`, {
columnName: builder.name,
columnType: type,
})
}
let render = `"${builder.name}" ${inverseFieldType(type)}`
if ( builder.getLength() ) {
render += `(${builder.getLength()})`
}
const defaultValue = builder.getDefaultValue()
if ( typeof defaultValue !== 'undefined' ) {
render += ` DEFAULT ${this.escape(defaultValue)}`
}
if ( builder.isPrimary() ) {
render += ` CONSTRAINT ${builder.name}_pk PRIMARY KEY`
}
if ( builder.isUnique() ) {
render += ` UNIQUE`
}
render += ` ${builder.isNullable() ? 'NULL' : 'NOT NULL'}`
return render
}
public renderDropTable(builder: TableBuilder): string {
return `DROP TABLE ${builder.isSkippedIfExisting() ? 'IF EXISTS ' : ''}${builder.name}`
}
public renderCreateIndex(builder: IndexBuilder): string {
const cols = builder.getFields().map(x => `"${x}"`)
const parts = [
`CREATE ${builder.isUnique() ? 'UNIQUE ' : ''}INDEX ${builder.isSkippedIfExisting() ? 'IF NOT EXISTS ' : ''}${builder.name}`,
` ON ${builder.parent.name}`,
` (${cols.join(',')})`,
]
return parts.join('\n')
}
public renderAlterTable(builder: TableBuilder): string {
const alters: string[] = []
const columns = builder.getColumns()
for ( const columnName in columns ) {
if ( !Object.prototype.hasOwnProperty.call(columns, columnName) ) {
continue
}
const columnBuilder = columns[columnName]
if ( !columnBuilder.isExisting() ) {
// The column doesn't exist on the table, but was added to the schema
alters.push(` ADD COLUMN ${this.renderColumnDefinition(columnBuilder)}`)
} else if ( columnBuilder.isDirty() && columnBuilder.originalFromSchema ) {
// The column exists in the table, but was modified in the schema
if ( columnBuilder.isDropping() || columnBuilder.isDroppingIfExists() ) {
alters.push(` DROP COLUMN "${columnBuilder.name}"`)
continue
}
// Change the data type of the column
if ( columnBuilder.getType() !== columnBuilder.originalFromSchema.getType() ) {
const renderedType = `${columnBuilder.getType()}${columnBuilder.getLength() ? `(${columnBuilder.getLength()})` : ''}`
alters.push(` ALTER COLUMN "${columnBuilder.name}" TYPE ${renderedType}`)
}
// Change the default value of the column
if ( columnBuilder.getDefaultValue() !== columnBuilder.originalFromSchema.getDefaultValue() ) {
alters.push(` ALTER COLUMN "${columnBuilder.name}" SET default ${this.escape(columnBuilder.getDefaultValue())}`)
}
// Change the nullable-status of the column
if ( columnBuilder.isNullable() !== columnBuilder.originalFromSchema.isNullable() ) {
if ( columnBuilder.isNullable() ) {
alters.push(` ALTER COLUMN "${columnBuilder.name}" DROP NOT NULL`)
} else {
alters.push(` ALTER COLUMN "${columnBuilder.name}" SET NOT NULL`)
}
}
// Change the name of the column
if ( columnBuilder.getRename() ) {
alters.push(` RENAME COLUMN "${columnBuilder.name}" TO "${columnBuilder.getRename()}"`)
}
}
}
const constraints = builder.getConstraints()
for ( const constraintName in constraints ) {
if ( !Object.prototype.hasOwnProperty.call(constraints, constraintName) ) {
continue
}
const constraintBuilder = constraints[constraintName]
// Drop the constraint if specified
if ( constraintBuilder.isDropping() ) {
alters.push(` DROP CONSTRAINT ${constraintBuilder.name}`)
continue
}
// Drop the constraint with IF EXISTS if specified
if ( constraintBuilder.isDroppingIfExists() ) {
alters.push(` DROP CONSTRAINT IF EXISTS ${constraintBuilder.name}`)
continue
}
// Otherwise, drop and recreate the constraint if it was modified
if ( constraintBuilder.isDirty() ) {
if ( constraintBuilder.isExisting() ) {
alters.push(` DROP CONSTRAINT IF EXISTS ${constraintBuilder.name}`)
}
const constraintDefinition = this.renderConstraintDefinition(constraintBuilder)
if ( constraintDefinition ) {
alters.push(` ADD CONSTRAINT ${constraintDefinition}`)
}
}
}
if ( builder.getRename() ) {
alters.push(` RENAME TO "${builder.getRename()}"`)
}
return 'ALTER TABLE ' + builder.name + '\n' + alters.join(',\n')
}
public renderDropIndex(builder: IndexBuilder): string {
return `DROP INDEX ${builder.isDroppingIfExists() ? 'IF EXISTS ' : ''}${builder.name}`
}
public renderTransaction(queries: string[]): string {
const parts = [
'BEGIN',
...queries,
'COMMIT',
]
return parts.join(';\n\n')
}
public renderRenameIndex(builder: IndexBuilder): string {
return `ALTER INDEX ${builder.name} RENAME TO ${builder.getRename()}`
}
public renderRecreateIndex(builder: IndexBuilder): string {
return `${this.renderDropIndex(builder)};\n\n${this.renderCreateIndex(builder)}`
}
public renderDropColumn(builder: ColumnBuilder): string {
const parts = [
`ALTER TABLE ${builder.parent.name} ${builder.parent.isSkippedIfExisting() ? 'IF EXISTS ' : ''}`,
` DROP COLUMN ${builder.isSkippedIfExisting() ? 'IF EXISTS ' : ''}${builder.name}`,
]
return parts.join('\n')
}
}

132
src/orm/dialect/SQLDialect.ts

@ -1,6 +1,7 @@
import {Constraint} from '../types'
import {AbstractBuilder} from '../builder/AbstractBuilder'
import {AppClass} from '../../lifecycle/AppClass'
import {ColumnBuilder, IndexBuilder, TableBuilder} from '../schema/TableBuilder'
/**
* A value which can be escaped to be interpolated into an SQL query.
@ -160,10 +161,141 @@ export abstract class SQLDialect extends AppClass {
* This function should escape the values before they are included in the query string.
*
* @example
* ```ts
* dialect.renderUpdateSet({field1: 'value', field2: 45})
* // => "SET field1 = 'value', field2 = 45"
* ```
*
* @param data
*/
public abstract renderUpdateSet(data: {[key: string]: EscapeValue}): string;
/**
* Given a table schema-builder, render a `CREATE TABLE...` query.
* @param builder
*/
public abstract renderCreateTable(builder: TableBuilder): string;
/**
* Given a table schema-builder, render an `ALTER TABLE...` query.
* @param builder
*/
public abstract renderAlterTable(builder: TableBuilder): string;
/**
* Given a table schema-builder, render a `DROP TABLE...` query.
* @param builder
*/
public abstract renderDropTable(builder: TableBuilder): string;
/**
* Render the table-column definitions for the table defined by
* the given schema-builder.
*
* @example
* ```ts
* dialect.renderTableColumns(builder)
* // => ['col1 varchar(100) NULL', 'col2 serial NOT NULL']
* ```
*
* @param builder
*/
public abstract renderTableColumns(builder: TableBuilder): string[];
/**
* Given an index schema-builder, render a `CREATE INDEX...` query.
* @param builder
*/
public abstract renderCreateIndex(builder: IndexBuilder): string;
/**
* Given a column schema-builder, render an `ALTER TABLE... DROP COLUMN...` query.
* @param builder
*/
public abstract renderDropColumn(builder: ColumnBuilder): string;
/**
* Given an index schema-builder, render a `DROP INDEX...` query.
* @param builder
*/
public abstract renderDropIndex(builder: IndexBuilder): string;
/**
* Given an index schema-builder, render an `ALTER INDEX... RENAME...` query.
* @param builder
*/
public abstract renderRenameIndex(builder: IndexBuilder): string;
/**
* Given an index schema-builder, render either an `ALTER INDEX...` query,
* or a `DROP INDEX...; CREATE INDEX...` query.
* @param builder
*/
public abstract renderRecreateIndex(builder: IndexBuilder): string;
/**
* Given a series of fully-formed queries, render them as a single transaction.
* @example
* ```ts
* const queries = [
* 'SELECT * FROM a',
* 'UPDATE b SET col = 123',
* ]
*
* dialect.renderTransaction(queries)
* // => 'BEGIN; SELECT * FROM a; UPDATE b SET col = 123; COMMIT;'
* ```
* @param queries
*/
public abstract renderTransaction(queries: string[]): string;
/**
* Given a table schema-builder, render a series of queries as a transaction
* that apply the given schema to database.
* @todo handle constraints better - ConstraintBuilder
* @param builder
*/
public renderCommitSchemaTransaction(builder: TableBuilder): string {
if ( builder.isDropping() || builder.isDroppingIfExists() ) {
// If we're dropping the table, just return the DROP TABLE query
return this.renderTransaction([
this.renderDropTable(builder),
])
}
// Render the queries to create/update/drop indexes
const indexes = Object.values(builder.getIndexes())
.filter(index => !index.isExisting() || index.isDirty())
.map(index => {
if ( index.isDropping() || index.isDroppingIfExists() ) {
return this.renderDropIndex(index)
}
if ( index.isExisting() ) {
// The index was changed in the schema, but exists in the DB
return this.renderRecreateIndex(index)
}
return this.renderCreateIndex(index)
})
// Render the queries to rename indexes AFTER the above operations
const renamedIndexes = Object.values(builder.getIndexes())
.filter(idx => idx.getRename())
.map(x => this.renderRenameIndex(x))
let parts: string[] = []
// Render the CREATE/ALTER TABLE query
if ( !builder.isExisting() && builder.isDirty() ) {
parts.push(this.renderCreateTable(builder))
} else if ( builder.isExisting() && builder.isDirty() ) {
parts.push(this.renderAlterTable(builder))
}
// Render the various schema queries as a single transaction
parts = parts.concat(...indexes)
parts = parts.concat(...renamedIndexes)
return this.renderTransaction(parts)
}
}

48
src/orm/directive/CreateMigrationDirective.ts

@ -0,0 +1,48 @@
import {Directive, OptionDefinition} from '../../cli'
import {Injectable} from '../../di'
import {stringToPascal} from '../../util'
import {templateMigration} from '../template/migration'
/**
* CLI directive that creates migration classes from template.
*/
@Injectable()
export class CreateMigrationDirective extends Directive {
getDescription(): string {
return 'create a new migration'
}
getKeywords(): string | string[] {
return ['create-migration', 'make-migration']
}
getOptions(): OptionDefinition[] {
return [
'{description} | Description of what the migration does',
]
}
getHelpText(): string {
return [
'Creates a new migration file in `src/app/migrations`.',
'To use, specify a string describing what the migration does. For example:',
'./ex create-migration "Add version column to sessions table"',
].join('\n\n')
}
async handle(): Promise<void> {
const description = this.option('description')
const className = `${stringToPascal(description)}Migration`
const fileName = `${(new Date()).toISOString()}_${className}.migration.ts`
const path = this.app().path('..', 'src', 'app', 'migrations', fileName)
// Create the migrations directory, if it doesn't already exist
await path.concat('..').mkdir()
// Render the template
const rendered = await templateMigration.render(className, className, path)
await path.write(rendered)
this.success(`Created migration: ${className}`)
}
}

117
src/orm/directive/MigrateDirective.ts

@ -0,0 +1,117 @@
import {Directive, OptionDefinition} from '../../cli'
import {Container, Inject, Injectable} from '../../di'
import {EventBus} from '../../event/EventBus'
import {Migrator} from '../migrations/Migrator'
import {Migrations} from '../services/Migrations'
import {ApplyingMigrationEvent} from '../migrations/events/ApplyingMigrationEvent'
import {AppliedMigrationEvent} from '../migrations/events/AppliedMigrationEvent'
import {EventSubscription} from '../../event/types'
import {NothingToMigrateError} from '../migrations/NothingToMigrateError'
/**
* CLI directive that applies migrations using the default Migrator.
* @fixme Support dry run mode
*/
@Injectable()
export class MigrateDirective extends Directive {
@Inject()
protected readonly bus!: EventBus
@Inject('injector')
protected readonly injector!: Container
/** Event bus subscriptions. */
protected subscriptions: EventSubscription[] = []
getKeywords(): string | string[] {
return ['migrate']
}
getDescription(): string {
return 'apply pending migrations'
}
getOptions(): OptionDefinition[] {
return [
'--package -p {name} | apply migrations for a specific namespace',
'--identifier -i {name} | apply a specific migration, by identifier',
]
}
getHelpText(): string {
return [
'Migrations are single-run code patches used to track changes to things like database schemata.',
'',
'You can create migrations in your app using the ./ex command and they can be applied and rolled-back.',
'',
'./ex migrate:create "Add version column to sessions table"',
'',
'Modules and packages can also register their own migrations. These are run by default.',
'',
'To run the migrations for a specific package, and no others, use the --package option. Example:',
'',
'./ex migrate --package @extollo',
'',
].join('\n')
}
async handle(): Promise<void> {
await this.registerListeners()
const namespace = this.option('package')
const identifier = this.option('identifier')
let identifiers
if ( namespace ) {
identifiers = (this.injector.make<Migrations>(Migrations))
.all(namespace)
.map(id => `${namespace}:${id}`)
}
if ( identifier ) {
if ( !identifiers ) {
identifiers = [identifier]
}
identifiers = identifiers.filter(x => x === identifier)
}
let error
try {
await (this.injector.make<Migrator>(Migrator)).migrate(identifiers)
} catch (e) {
if ( e instanceof NothingToMigrateError ) {
this.info(e.message)
} else {
error = e
this.error(e)
}
} finally {
await this.removeListeners()
}
if ( error ) {
throw error
}
}
/**
* Register event bus listeners to print messages for the user.
* @protected
*/
protected async registerListeners(): Promise<void> {
this.subscriptions.push(await this.bus.subscribe(ApplyingMigrationEvent, event => {
this.info(`Applying migration ${event.migration.identifier}...`)
}))
this.subscriptions.push(await this.bus.subscribe(AppliedMigrationEvent, event => {
this.success(`Applied migration: ${event.migration.identifier}`)
}))
}
/** Remove event bus listeners before finish. */
protected async removeListeners(): Promise<void> {
await Promise.all(this.subscriptions.map(x => x.unsubscribe()))
this.subscriptions = []
}
}

102
src/orm/directive/RollbackDirective.ts

@ -0,0 +1,102 @@
import {Directive, OptionDefinition} from '../../cli'
import {Container, Inject, Injectable} from '../../di'
import {EventBus} from '../../event/EventBus'
import {Migrator} from '../migrations/Migrator'
import {Migrations} from '../services/Migrations'
import {RollingBackMigrationEvent} from '../migrations/events/RollingBackMigrationEvent'
import {RolledBackMigrationEvent} from '../migrations/events/RolledBackMigrationEvent'
import {EventSubscription} from '../../event/types'
import {NothingToMigrateError} from '../migrations/NothingToMigrateError'
/**
* CLI directive that undoes applied migrations using the default Migrator.
* @fixme Support dry run mode
*/
@Injectable()
export class RollbackDirective extends Directive {
@Inject()
protected readonly bus!: EventBus
@Inject('injector')
protected readonly injector!: Container
@Inject()
protected readonly migrations!: Migrations
/** Event bus subscriptions. */
protected subscriptions: EventSubscription[] = []
getKeywords(): string | string[] {
return ['rollback']
}
getDescription(): string {
return 'roll-back applied migrations'
}
getOptions(): OptionDefinition[] {
return [
'--identifier -i {name} | roll-back a specific migration, by identifier',
]
}
getHelpText(): string {
return [
'Use this command to undo one or more migrations that were applied.',
'',
'By default, the command will undo all of the migrations applied the last time the migrate command was run.',
'',
'To undo a specific migration, pass its identifier using the --identifier option.',
'',
].join('\n')
}
async handle(): Promise<void> {
await this.registerListeners()
const identifier = this.option('identifier')
let identifiers
if ( identifier ) {
identifiers = [identifier]
}
let error
try {
await (this.injector.make<Migrator>(Migrator)).rollback(identifiers)
} catch (e) {
if ( e instanceof NothingToMigrateError ) {
this.info(e.message)
} else {
error = e
this.error(e)
}
} finally {
await this.removeListeners()
}
if ( error ) {
throw error
}
}
/**
* Register event-bus listeners to print messages for the user.
* @protected
*/
protected async registerListeners(): Promise<void> {
this.subscriptions.push(await this.bus.subscribe(RollingBackMigrationEvent, event => {
this.info(`Rolling-back migration ${event.migration.identifier}...`)
}))
this.subscriptions.push(await this.bus.subscribe(RolledBackMigrationEvent, event => {
this.success(`Rolled-back migration: ${event.migration.identifier}`)
}))
}
/** Remove event bus listeners before finish. */
protected async removeListeners(): Promise<void> {
await Promise.all(this.subscriptions.map(x => x.unsubscribe()))
this.subscriptions = []
}
}

25
src/orm/index.ts

@ -18,9 +18,6 @@ export * from './model/ModelResultIterable'
export * from './model/events'
export * from './model/Model'
export * from './services/Database'
export * from './services/Models'
export * from './support/SessionModel'
export * from './support/ORMSession'
export * from './support/CacheModel'
@ -28,3 +25,25 @@ export * from './support/ORMCache'
export * from './DatabaseService'
export * from './types'
export * from './schema/TableBuilder'
export * from './schema/Schema'
export * from './schema/PostgresSchema'
export * from './migrations/NothingToMigrateError'
export * from './migrations/events/ApplyingMigrationEvent'
export * from './migrations/events/AppliedMigrationEvent'
export * from './migrations/events/RollingBackMigrationEvent'
export * from './migrations/events/RolledBackMigrationEvent'
export * from './migrations/Migration'
export * from './migrations/Migrator'
export * from './migrations/MigratorFactory'
export * from './migrations/DatabaseMigrator'
export * from './services/Database'
export * from './services/Models'
export * from './services/Migrations'
export * from './directive/CreateMigrationDirective'
export * from './directive/MigrateDirective'
export * from './directive/RollbackDirective'

179
src/orm/migrations/DatabaseMigrator.ts

@ -0,0 +1,179 @@
import {Container, Inject, Injectable} from '../../di'
import {Migrator} from './Migrator'
import {DatabaseService} from '../DatabaseService'
import {FieldType} from '../types'
import {Migration} from './Migration'
import {Builder} from '../builder/Builder'
/**
* Migrator implementation that tracks applied migrations in a database table.
* @todo allow configuring more of this
*/
@Injectable()
export class DatabaseMigrator extends Migrator {
@Inject()
protected readonly db!: DatabaseService
@Inject('injector')
protected readonly injector!: Container
/** True if we've initialized the migrator. */
protected initialized = false
public async initialize(): Promise<void> {
await super.initialize()
if ( this.initialized ) {
return
}
const schema = this.db.get().schema()
if ( !(await schema.hasTable('migrations')) ) {
const table = await schema.table('migrations')
table.primaryKey('id', FieldType.serial).required()
table.column('identifier')
.type(FieldType.varchar)
.required()
table.column('applygroup')
.type(FieldType.integer)
.required()
table.column('applydate')
.type(FieldType.timestamp)
.required()
await schema.commit(table)
}
this.initialized = true
}
async has(migration: Migration): Promise<boolean> {
return this.builder()
.connection('default')
.select('id')
.from('migrations')
.where('identifier', '=', migration.identifier)
.exists()
}
async markApplied(migrations: Migration | Migration[], applyDate: Date = new Date()): Promise<void> {
if ( !Array.isArray(migrations) ) {
migrations = [migrations]
}
const applyGroup = await this.getNextGroupIdentifier()
const rows = migrations.map(migration => {
return {
applygroup: applyGroup,
applydate: applyDate,
identifier: migration.identifier,
}
})
await this.builder()
.connection('default')
.table('migrations')
.insert(rows)
}
async unmarkApplied(migrations: Migration | Migration[]): Promise<void> {
if ( !Array.isArray(migrations) ) {
migrations = [migrations]
}
const identifiers = migrations.map(migration => migration.identifier)
await this.builder()
.connection('default')
.table('migrations')
.whereIn('identifier', identifiers)
.delete()
}
async getLastApplyGroup(): Promise<string[]> {
const applyGroup = await this.builder()
.connection('default')
.select('applygroup')
.from('migrations')
.get()
.max<number>('applygroup')
return this.builder()
.connection('default')
.select('identifier')
.from('migrations')
.where('applygroup', '=', applyGroup)
.get()
.asyncPipe()
.tap(coll => {
return coll.pluck<string>('identifier')
})
.tap(coll => {
return coll.all()
})
.resolve()
}
/**
* Helper method to look up the next `applygroup` that should be used.
* @protected
*/
protected async getNextGroupIdentifier(): Promise<number> {
const current = await this.builder()
.connection('default')
.select('applygroup')
.from('migrations')
.get()
.max<number>('applygroup')
return (current ?? 0) + 1
}
/**
* Given a list of migration identifiers, filter out those that have been applied.
* @override to make this more efficient
* @param identifiers
* @protected
*/
protected async filterAppliedMigrations(identifiers: string[]): Promise<string[]> {
const existing = await this.builder()
.connection('default')
.select('identifier')
.from('migrations')
.whereIn('identifier', identifiers)
.get()
.pluck<string>('identifier')
return identifiers.filter(id => !existing.includes(id))
}
/**
* Given a list of migration identifiers, filter out those that have not been applied.
* @override to make this more efficient
* @param identifiers
* @protected
*/
protected async filterPendingMigrations(identifiers: string[]): Promise<string[]> {
const existing = await this.builder()
.connection('default')
.select('identifier')
.from('migrations')
.whereIn('identifier', identifiers)
.get()
.pluck<string>('identifier')
return existing.all()
}
/**
* Get a query builder instance.
* @protected
*/
protected builder(): Builder {
return this.injector.make<Builder>(Builder)
}
}

39
src/orm/migrations/Migration.ts

@ -0,0 +1,39 @@
import {Injectable} from '../../di'
import {Awaitable} from '../../util'
/**
* Abstract base-class for one-time migrations.
*/
@Injectable()
export abstract class Migration {
/** Set by the Migrations unit on load. */
protected migrationIdentifier!: string
/**
* Sets the migration identifier.
* This is used internally when the Migrations service loads
* the migration files to determine the ID from the file-name.
* It shouldn't be used externally.
* @param name
*/
public setMigrationIdentifier(name: string): void {
this.migrationIdentifier = name
}
/**
* Get the unique identifier of this migration.
*/
public get identifier(): string {
return this.migrationIdentifier
}
/**
* Apply the migration.
*/
abstract up(): Awaitable<void>
/**
* Undo the migration.
*/
abstract down(): Awaitable<void>
}

295
src/orm/migrations/Migrator.ts

<
@ -0,0 +1,295 @@
import {Container, Inject, Injectable} from '../../di'
import {Awaitable, collect, ErrorWithContext} from '../../util'
import {Migration} from './Migration'
import {Migrations} from '../services/Migrations'
import {EventBus} from '../../event/EventBus'
import {ApplyingMigrationEvent} from './events/ApplyingMigrationEvent'
import {AppliedMigrationEvent} from './events/AppliedMigrationEvent'
import {RollingBackMigrationEvent} from './events/RollingBackMigrationEvent'
import {RolledBackMigrationEvent} from './events/RolledBackMigrationEvent'
import {NothingToMigrateError} from './NothingToMigrateError'
/**
* Manages single-run patches/migrations.
*/
@Injectable()
export abstract class Migrator {
@Inject()
protected readonly migrations!: Migrations
@Inject()
protected readonly bus!: EventBus
@Inject('injector')
protected readonly injector!: Container
/**
* Should resolve true if the given migration has already been applied.
* @param migration
*/
public abstract has(migration: Migration): Awaitable<boolean>
/**
* Should mark the given migrations as being applied.
*
* If a date is specified, then that is the timestamp when the migrations
* were applied, otherwise, use `new Date()`.
*
* @param migrations
* @param date
*/
public abstract markApplied(migrations: Migration | Migration[], date?: Date): Awaitable<void>
/**
* Should un-mark the given migrations as being applied.
* @param migration
*/
public abstract unmarkApplied(migration: Migration | Migration[]): Awaitable<void>
/**
* Get the identifiers of the last group of migrations that were applied.
*/
public abstract getLastApplyGroup(): Awaitable<string[]>
/**
* Do any initial setup required to get the migrator ready.
* This can be overridden by implementation classes to do any necessary setup.
*/
public initialize(): Awaitable<void> {} // eslint-disable-line @typescript-eslint/no-empty-function
/**
* Apply pending migrations.
*
* If identifiers are specified, only the pending migrations with those
* identifiers are applied. If none are specified, all pending migrations
* will be applied.
*
* @param identifiers
*/
public async migrate(identifiers?: string[]): Promise<void> {
await this.initialize()
if ( !identifiers ) {
identifiers = this.getAllMigrationIdentifiers()
}
identifiers = (await this.filterAppliedMigrations(identifiers)).sort()
if ( !identifiers.length ) {
throw new NothingToMigrateError()
}
const migrations = collect(identifiers)
.map(id => {
const migration = this.migrations.get(id)
if ( !migration ) {
throw new ErrorWithContext(`Unable to find migration with identifier: ${id}`, {
identifier: id,
})
}
return migration
})
await migrations.promiseMap(migration => {
return this.apply(migration)
})
await this.markApplied(migrations.all())
}
/**
* Rollback applied migrations.
*
* If specified, only applied migrations with the given identifiers will
* be rolled back. If not specified, then the last "batch" of applied
* migrations will be rolled back.
*
* @param identifiers
*/
public async rollback(identifiers?: string[]): Promise<void> {
await this.initialize()
if ( !identifiers ) {
identifiers = await this.getLastApplyGroup()
}
identifiers = (await this.filterPendingMigrations(identifiers)).sort()
if ( !identifiers.length ) {
throw new NothingToMigrateError()
}
const migrations = collect(identifiers)
.map(id => {
const migration = this.migrations.get(id)
if ( !migration ) {
throw new ErrorWithContext(`Unable to find migration with identifier: ${id}`, {
identifier: id,
})
}
return migration
})
await migrations.promiseMap(migration => {
return this.undo(migration)
})
await this.unmarkApplied(migrations.all())
}
/**
* Apply a single migration.
* @param migration
*/
public async apply(migration: Migration): Promise<void> {
await this.initialize()
await this.applying(migration)
await migration.up()
await this.applied(migration)
}
/**
* Rollback a single migration.
* @param migration