Create initial standup logic to bootstrap control CT & install Node.js

.gitignore

@@ -193,3 +193,5 @@ dist
 /exbuild
 
 *.sqlite
+notes.md
+src/dev_tools.ts

package.json

@@ -9,17 +9,26 @@
   },
   "dependencies": {
     "@atao60/fse-cli": "^0.1.7",
-    "@extollo/lib": "^0.14.10",
+    "@extollo/lib": "^0.14.13",
     "@types/node": "^18.15.11",
+    "@types/ssh2": "^1.11.13",
+    "@types/sshpk": "^1.17.1",
+    "@types/ws": "^8.5.5",
     "copyfiles": "^2.4.1",
+    "engine.io-client": "^6.5.1",
     "feed": "^4.2.2",
     "gotify": "^1.1.0",
+    "ip-cidr": "^3.1.0",
+    "proxmox-api": "^1.0.0",
     "rimraf": "^3.0.2",
+    "ssh2": "^1.14.0",
+    "sshpk": "^1.17.0",
     "ts-expose-internals": "^4.5.4",
     "ts-node": "^10.9.0",
     "ts-patch": "^2.0.1",
     "ts-to-zod": "^1.8.0",
     "typescript": "^4.3.2",
+    "ws": "^8.13.0",
     "zod": "^3.11.6"
   },
   "scripts": {

src/app/AppUnit.ts

@@ -1,9 +1,14 @@
-import {ORMUser, Singleton, Unit} from '@extollo/lib'
+import {ORMUser, Singleton, Unit, CommandLine, Inject} from '@extollo/lib'
 import {User} from './models/User.model'
+import {StandupDirective} from './cli/directive/StandupDirective'
 
 @Singleton()
 export class AppUnit extends Unit {
+    @Inject()
+    protected readonly cli!: CommandLine
+
     async up(): Promise<void> {
         this.container().registerStaticOverride(ORMUser, User)
+        this.cli.registerDirective(StandupDirective)
     }
 }

src/app/cli/directive/StandupDirective.ts

@@ -0,0 +1,135 @@
+import {Directive, Inject, Injectable, OptionDefinition, uuid4} from '@extollo/lib'
+import { IpRange } from "../../models/IpRange.model";
+import { Setting } from "../../models/Setting.model";
+import { Provisioner } from "../../services/Provisioner.service";
+import { IpAddress, isIpAddress, isSubnet, Subnet } from "../../types";
+import * as crypto from 'crypto'
+
+@Injectable()
+export class StandupDirective extends Directive {
+    @Inject()
+    protected readonly provisioner!: Provisioner
+
+    getKeywords(): string | string[] {
+        return 'standup'
+    }
+
+    getDescription(): string {
+        return 'perform initial configuration and setup for the control server'
+    }
+
+    getOptions(): OptionDefinition[] {
+        return [
+            '--pve-master-node {name} | the name of the master node in the PVE cluster',
+            '--pve-api-host {host} | the hostname or IP of a PVE cluster node',
+            // '--pve-token-id {id} | the fully-qualified ID of a PVE API token',
+            // '--pve-token-secret {secret} | secret for the PVE API token',
+            '--pve-root-password {password} | password for the root@pam PVE user',
+            '--ip-pool-start {start} | the starting address in the P5X IP pool',
+            '--ip-pool-end {end} | the final address in the P5X IP pool',
+            '--ip-pool-subnet {subnet} | the numeric subnet of the P5X IP pool (e.g. 24)',
+            '--ip-pool-gateway {gateway} | the address of the gateway for the P5X IP pool',
+            '--storage-pool {name} | the name of the PVE shared storage pool',
+            '--dns-domain {name} | the DNS domain name used for K8s nodes',
+            '--node-network-bridge {name} | the name of the PVE network bridge to which nodes will join',
+            '--node-cpu-cores {cores} | the number of cores to provision each K8s node with',
+            '--node-ram-mib {ram} | the amount of ram (in MiB) to provision each K8s node with',
+        ]
+    }
+
+    public async handle(argv: string[]): Promise<void> {
+        // Configure the settings:
+        await Setting.set('pveMasterNode', `${this.getOptionOrFail('pve-master-node')}`)
+        await Setting.set('pveApiHost', `${this.getOptionOrFail('pve-api-host')}`)
+        // await Setting.set('pveTokenId', `${this.getOptionOrFail('pve-token-id')}`)
+        // await Setting.set('pveTokenSecret', `${this.getOptionOrFail('pve-token-secret')}`)
+        await Setting.set('pveRootPassword', `${this.getOptionOrFail('pve-root-password')}`)
+        await Setting.set('pveStoragePool', `${this.getOptionOrFail('storage-pool')}`)
+        await Setting.set('dnsDomain', `${this.getOptionOrFail('dns-domain')}`)
+        await Setting.set('nodeNetworkBridge', `${this.getOptionOrFail('node-network-bridge')}`)
+        await Setting.set('nodeCpus', this.getIntOptionOrFail('node-cpu-cores'))
+        await Setting.set('nodeRamMib', this.getIntOptionOrFail('node-ram-mib'))
+        await Setting.set('rootPassword', uuid4())
+
+        const [sshPublicKey, sshPrivateKey] = await this.getSSHKeyPair()
+        await Setting.set('sshPublicKey', sshPublicKey)
+        await Setting.set('sshPrivateKey', sshPrivateKey)
+
+        // Store the IP address pool:
+        const ipRange = this.container().makeNew<IpRange>(IpRange)
+        ipRange.name = 'Default address pool'
+        ipRange.startIp = this.getIpOptionOrFail('ip-pool-start')
+        ipRange.endIp = this.getIpOptionOrFail('ip-pool-end')
+        ipRange.subnet = this.getSubnetOptionOrFail('ip-pool-subnet')
+        ipRange.gatewayIp = this.getIpOptionOrFail('ip-pool-gateway')
+        ipRange.save()
+
+        // Provision the master node:
+        const masterNode = await this.provisioner.provisionMasterNode()
+    }
+
+    protected getIntOptionOrFail(option: string): number {
+        const val = parseInt(this.option(option, NaN), 10)
+        if ( isNaN(val) ) {
+            this.error(`Missing or malformed required integer option: --${option}`)
+            process.exit(1)
+        }
+        return val
+    }
+
+    protected getSubnetOptionOrFail(option: string): Subnet {
+        const val = parseInt(this.option(option, NaN), 10)
+        if ( !isSubnet(val) ) {
+            this.error(`Missing or malformed required numeric subnet: --${option}`)
+            process.exit(1)
+        }
+        return val
+    }
+
+    protected getIpOptionOrFail(option: string): IpAddress {
+        const val = `${this.option(option, '')}`
+        if ( !isIpAddress(val) ) {
+            this.error(`Missing or malformed required IP address: --${option}`)
+            process.exit(1)
+        }
+        return val
+    }
+
+    protected getOptionOrFail(option: string): unknown {
+        const val = this.option(option, undefined)
+        if ( typeof val === 'undefined' ) {
+            this.error(`Missing required option: --${option}`)
+            process.exit(1)
+        }
+        return val
+    }
+
+    protected async getSSHKeyPair(): Promise<[string, string]> {
+        return new Promise<[string, string]>((res, rej) => {
+            crypto.generateKeyPair(
+                'rsa',
+                {
+                    modulusLength: 4096,
+                    publicKeyEncoding: {
+                        type: 'spki',
+                        format: 'pem',
+                    },
+                    privateKeyEncoding: {
+                        type: 'pkcs8',
+                        format: 'pem',
+                        cipher: 'aes-256-cbc',
+                        passphrase: '',
+                    },
+                },
+                (err, pubkey, privkey) => {
+                    if ( err ) {
+                        return rej(err)
+                    }
+
+                    res([pubkey, privkey])
+                },
+            )
+        })
+    }
+
+}

src/app/migrations/2023-04-01T03:01:02.528Z_CreateP5xCredentialsTableMigration.migration.ts

@@ -0,0 +1,48 @@
+import {Injectable, Migration, Inject, DatabaseService, FieldType} from '@extollo/lib'
+
+/**
+ * CreateP5xCredentialsTableMigration
+ * ----------------------------------
+ * Create a table to store the PVE user credentials.
+ */
+@Injectable()
+export default class CreateP5xCredentialsTableMigration extends Migration {
+    @Inject()
+    protected readonly db!: DatabaseService
+
+    /**
+     * Apply the migration.
+     */
+    async up(): Promise<void> {
+        const schema = this.db.get().schema()
+        const table = await schema.table('p5x_credentials')
+
+        table.primaryKey('id')
+
+        table.column('pve_username')
+            .type(FieldType.varchar)
+            .required()
+
+        table.column('pve_credential')
+            .type(FieldType.text)
+            .required()
+
+        table.column('is_api_token')
+            .type(FieldType.bool)
+            .required()
+
+        await schema.commit(table)
+    }
+
+    /**
+     * Undo the migration.
+     */
+    async down(): Promise<void> {
+        const schema = this.db.get().schema()
+        const table = await schema.table('p5x_credentials')
+
+        table.dropIfExists()
+
+        await schema.commit(table)
+    }
+}

src/app/migrations/2023-04-01T03:02:21.621Z_CreateP5xIpRangesTableMigration.migration.ts

@@ -0,0 +1,56 @@
+import {Injectable, Migration, Inject, DatabaseService, FieldType} from '@extollo/lib'
+
+/**
+ * CreateP5xIpRangesTableMigration
+ * ----------------------------------
+ * Put some description here.
+ */
+@Injectable()
+export default class CreateP5xIpRangesTableMigration extends Migration {
+    @Inject()
+    protected readonly db!: DatabaseService
+
+    /**
+     * Apply the migration.
+     */
+    async up(): Promise<void> {
+        const schema = this.db.get().schema()
+        const table = await schema.table('p5x_ip_ranges')
+
+        table.primaryKey('id')
+
+        table.column('name')
+            .type(FieldType.varchar)
+            .required()
+
+        table.column('start_ip')
+            .type(FieldType.varchar)
+            .required()
+
+        table.column('end_ip')
+            .type(FieldType.varchar)
+            .required()
+
+        table.column('subnet')
+            .type(FieldType.int)
+            .required()
+
+        table.column('gateway_ip')
+            .type(FieldType.varchar)
+            .required()
+
+        await schema.commit(table)
+    }
+
+    /**
+     * Undo the migration.
+     */
+    async down(): Promise<void> {
+        const schema = this.db.get().schema()
+        const table = await schema.table('p5x_ip_ranges')
+
+        table.dropIfExists()
+
+        await schema.commit(table)
+    }
+}

src/app/migrations/2023-04-01T03:28:47.961Z_CreateP5xNodesTableMigration.migration.ts

@@ -0,0 +1,62 @@
+import {Injectable, Migration, Inject, DatabaseService, FieldType} from '@extollo/lib'
+
+/**
+ * CreateP5xNodesTableMigration
+ * ----------------------------------
+ * Create a table to track the virtual K8s containers under our control.
+ */
+@Injectable()
+export default class CreateP5xNodesTableMigration extends Migration {
+    @Inject()
+    protected readonly db!: DatabaseService
+
+    /**
+     * Apply the migration.
+     */
+    async up(): Promise<void> {
+        const schema = this.db.get().schema()
+        const table = await schema.table('p5x_nodes')
+
+        table.primaryKey('id')
+
+        table.column('pve_id')
+            .type(FieldType.int)
+            .required()
+
+        table.column('hostname')
+            .type(FieldType.varchar)
+            .required()
+
+        table.column('assigned_ip')
+            .type(FieldType.varchar)
+            .required()
+
+        table.column('assigned_subnet')
+            .type(FieldType.int)
+            .required()
+
+        table.column('is_permanent')
+            .type(FieldType.bool)
+            .default(false)
+            .required()
+
+        table.column('is_master')
+            .type(FieldType.bool)
+            .default(false)
+            .required()
+
+        await schema.commit(table)
+    }
+
+    /**
+     * Undo the migration.
+     */
+    async down(): Promise<void> {
+        const schema = this.db.get().schema()
+        const table = await schema.table('p5x_nodes')
+
+        table.dropIfExists()
+
+        await schema.commit(table)
+    }
+}

src/app/migrations/2023-04-01T04:21:49.411Z_CreateSettingsTableMigration.migration.ts

@@ -0,0 +1,44 @@
+import {Injectable, Migration, Inject, DatabaseService, FieldType} from '@extollo/lib'
+
+/**
+ * CreateSettingsTableMigration
+ * ----------------------------------
+ * Create a table to store random one-off settings.
+ */
+@Injectable()
+export default class CreateSettingsTableMigration extends Migration {
+    @Inject()
+    protected readonly db!: DatabaseService
+
+    /**
+     * Apply the migration.
+     */
+    async up(): Promise<void> {
+        const schema = this.db.get().schema()
+        const table = await schema.table('p5x_settings')
+
+        table.primaryKey('id')
+
+        table.column('setting_key')
+            .type(FieldType.varchar)
+            .required()
+
+        table.column('setting_json')
+            .type(FieldType.text)
+            .required()
+
+        await schema.commit(table)
+    }
+
+    /**
+     * Undo the migration.
+     */
+    async down(): Promise<void> {
+        const schema = this.db.get().schema()
+        const table = await schema.table('p5x_settings')
+
+        table.dropIfExists()
+
+        await schema.commit(table)
+    }
+}

src/app/models/Credential.model.ts

@@ -0,0 +1,24 @@
+import {Injectable, Model, Field, FieldType} from '@extollo/lib'
+
+/**
+ * Credential Model
+ * -----------------------------------
+ * A username and {password|api token} for the PVE cluster.
+ */
+@Injectable()
+export class Credential extends Model<Credential> {
+    protected static table = 'p5x_credentials'
+    protected static key = 'id'
+
+    @Field(FieldType.serial)
+    public id?: number
+
+    @Field(FieldType.varchar, 'pve_username')
+    public username!: string
+
+    @Field(FieldType.text, 'pve_credential')
+    public credential!: string
+
+    @Field(FieldType.bool, 'is_api_token')
+    public isApiToken!: boolean
+}

src/app/models/IpRange.model.ts

@@ -0,0 +1,66 @@
+import {Injectable, Model, Field, FieldType, TypeTag, Maybe, collect, ErrorWithContext} from '@extollo/lib'
+import * as IPCIDR from 'ip-cidr'
+import {IpAddress, Subnet} from '../types'
+import {Node} from './Node.model'
+
+/**
+ * IpRange Model
+ * -----------------------------------
+ * Represents an available IP address pool from which we can allocate node addresses.
+ */
+@Injectable()
+export class IpRange extends Model<IpRange> {
+    protected static table = 'p5x_ip_ranges'
+    protected static key = 'id'
+
+    public static async getDefault(): Promise<IpRange> {
+        const ipRange = await this.query<IpRange>().first()
+        if ( !ipRange ) {
+            throw new Error('Unable to find default IP range!')
+        }
+        return ipRange
+    }
+
+    @Field(FieldType.serial)
+    public id?: number
+
+    @Field(FieldType.varchar)
+    public name!: string
+
+    @Field(FieldType.varchar, 'start_ip')
+    public startIp!: IpAddress
+
+    @Field(FieldType.varchar, 'end_ip')
+    public endIp!: IpAddress
+
+    @Field(FieldType.int)
+    public subnet!: Subnet
+
+    @Field(FieldType.varchar, 'gateway_ip')
+    public gatewayIp!: IpAddress
+
+    /** Get the next IP address in this range that has not already been allocated to a node. */
+    public async getNextAvailable(): Promise<Maybe<IpAddress>> {
+        // Generate the address pool
+        const cidr = new IPCIDR(`${this.gatewayIp}/${this.subnet}`)
+        const possibleIps = collect(cidr.toArray({ from: this.startIp, to: this.endIp }) as IpAddress[])
+            .filterOut(x => x.endsWith('.0'))
+
+        // Look up already-allocated addresses
+        const allocatedIps = await Node.query<Node>().get().pluck('assignedIp')
+
+        // Find the first available address
+        return possibleIps.firstWhere(ip => !allocatedIps.includes(ip))
+    }
+
+    /** Like `getNextAvailable(...)`, but it will throw if the range is exhausted */
+    public async getNextAvailableOrFail(): Promise<IpAddress> {
+        const ip = await this.getNextAvailable()
+        if ( !ip ) {
+            throw new ErrorWithContext('Unable to get next available IP from range: is range exhausted?', {
+                range: this.toObject(),
+            })
+        }
+        return ip
+    }
+}

src/app/models/Node.model.ts

@@ -0,0 +1,61 @@
+import {Injectable, Model, Field, FieldType} from '@extollo/lib'
+import {IpAddress, Subnet} from '../types'
+import {Host, SSHHost} from '../support/hosts'
+import * as ssh2 from 'ssh2'
+import * as sshpk from 'sshpk'
+import {Setting} from './Setting.model'
+import {PCTHost} from '../support/hosts/PCTHost'
+
+/**
+ * Node Model
+ * -----------------------------------
+ * Represents a single LXC container running a K8s worker node.
+ */
+@Injectable()
+export class Node extends Model<Node> {
+    protected static table = 'p5x_nodes'
+    protected static key = 'node_id'
+
+    @Field(FieldType.serial)
+    public id?: number
+
+    @Field(FieldType.varchar, 'pve_id')
+    public pveId!: number
+
+    @Field(FieldType.varchar)
+    public hostname!: string
+
+    @Field(FieldType.varchar, 'assigned_ip')
+    public assignedIp!: IpAddress
+
+    @Field(FieldType.int, 'assigned_subnet')
+    public assignedSubnet!: Subnet
+
+    @Field(FieldType.bool, 'is_permanent')
+    public isPermanent: boolean = false
+
+    @Field(FieldType.bool, 'is_master')
+    public isMaster: boolean = false
+
+    public async getHost(): Promise<Host> {
+        // Try to make a direct SSH connection to the container
+        const privKey = await Setting.loadOneRequired('sshPrivateKey')
+        const formattedPrivKey = sshpk.parsePrivateKey(privKey, 'pem', {passphrase: ''}).toString('ssh')
+        const directHost = this.container().makeNew<Host>(SSHHost, {
+            host: this.assignedIp,
+            username: 'root',
+            privateKey: formattedPrivKey,
+        } as ssh2.ConnectConfig)
+
+        if ( await directHost.isAlive() ) {
+            return directHost
+        }
+
+        // Otherwise, fall back to the PCTHost proxy
+        return this.container().makeNew<Host>(PCTHost, {
+            host: await Setting.loadOneRequired('pveApiHost'),
+            username: 'root',
+            password: await Setting.loadOneRequired('pveRootPassword'),
+        }, this.pveId)
+    }
+}

src/app/models/Setting.model.ts

@@ -0,0 +1,80 @@
+import {Injectable, Model, Field, FieldType, Collection, ErrorWithContext, make, Application} from '@extollo/lib'
+import { JSONString, Settings } from '../types';
+
+export class MissingRequiredSettingError extends ErrorWithContext {
+    constructor(key: string) {
+        super(`Missing required setting: ${key}`, { key })
+    }
+}
+
+/**
+ * Setting Model
+ * -----------------------------------
+ * Helper for loading various one-off settings.
+ */
+@Injectable()
+export class Setting<Key extends keyof Settings> extends Model<Setting<Key>> {
+    protected static table = 'p5x_settings'
+    protected static key = 'id'
+
+    public static load<Key extends keyof Settings>(key: Key): Promise<Collection<Settings[Key]>> {
+        return Setting.query<Setting<Key>>()
+            .where('setting_key', '=', key)
+            .get()
+            .map(x => x.value)
+    }
+
+    public static async loadRequired<Key extends keyof Settings>(key: Key): Promise<Collection<Settings[Key]>> {
+        const load = await this.load(key)
+        if ( load.isEmpty() ) {
+            throw new MissingRequiredSettingError(key)
+        }
+        return load
+    }
+
+    public static loadOneRequired<Key extends keyof Settings>(key: Key): Promise<Settings[Key]> {
+        return this.loadRequired(key).then(x => x.first()!)
+    }
+
+    public static async set<Key extends keyof Settings>(key: Key, value: Settings[Key]): Promise<void> {
+        // Delete any existing records w/ this key
+        await Setting.query().where('settingKey', '=', key).delete()
+
+        // Insert a new one
+        const setting = Application.getContainer().makeNew<Setting<Key>>(Setting);
+        setting.settingKey = key
+        setting.value = value
+        await setting.save()
+    }
+
+    public static async add<Key extends keyof Settings>(key: Key, value: Settings[Key]): Promise<void> {
+        // Delete any existing records w/ this key and value
+        await Setting.query()
+            .where('settingKey', '=', key)
+            .where('settingValue', '=', JSON.stringify(value))
+            .delete()
+
+        // Insert a new one
+        const setting = new Setting<Key>;
+        setting.settingKey = key
+        setting.value = value
+        await setting.save()
+    }
+
+    @Field(FieldType.serial)
+    protected id?: number
+
+    @Field(FieldType.varchar, 'setting_key')
+    protected settingKey!: Key
+
+    @Field(FieldType.text, 'setting_json')
+    protected settingJson!: JSONString
+
+    public get value(): Settings[Key] {
+        return JSON.parse(this.settingJson)
+    }
+
+    public set value(v: Settings[Key]) {
+        this.settingJson = JSON.stringify(v) as JSONString
+    }
+}

src/app/services/Images.service.ts

@@ -0,0 +1,38 @@
+import {appPath, AsyncCollection, collect, Collection, Singleton, UniversalPath, universalPath} from '@extollo/lib'
+
+export interface LXCImage {
+    os: string
+    version: string
+    arch: 'amd64' | 'arm64' | 'i386' | string
+    format: 'cloud' | 'default'
+    timestamp: string
+    endpoint: string
+}
+
+@Singleton()
+export class Images {
+    private imageServer = 'https://us.lxd.images.canonical.com'
+    private listEndpoint = 'meta/1.0/index-system'
+    private imageBinaryName = 'rootfs.tar.xz'
+
+    public async getImages(): Promise<Collection<LXCImage>> {
+        const url = universalPath(this.imageServer, this.listEndpoint)
+        const content = await fetch(url.toRemote).then(x => x.text())
+        return collect(content.trim().split('\n'))
+            .map(row => {
+                const [os, version, arch, format, timestamp, endpoint] = row.split(';')
+                return {
+                    os,
+                    version,
+                    arch,
+                    format,
+                    timestamp,
+                    endpoint,
+                } as LXCImage
+            })
+    }
+
+    public getImageBinaryPath(image: LXCImage): UniversalPath {
+        return universalPath(this.imageServer, image.endpoint, this.imageBinaryName)
+    }
+}

src/app/services/Provisioner.service.ts

@@ -0,0 +1,228 @@
+import {
+    Singleton,
+    Inject,
+    ErrorWithContext,
+    objectToKeyValue,
+    fetch,
+    Container,
+    Logging,
+    universalPath,
+} from '@extollo/lib'
+import { Node } from "../models/Node.model";
+import {Setting} from '../models/Setting.model'
+import {IpRange} from '../models/IpRange.model'
+import { Proxmox } from 'proxmox-api'
+import {Images, LXCImage} from './Images.service'
+import {unsafeESMImport} from '@extollo/lib/lib/util/unsafe'
+import * as sshpk from 'sshpk'
+import {shellCommand, SSHHost} from '../support/hosts'
+
+@Singleton()
+export class Provisioner {
+    @Inject()
+    protected readonly images!: Images
+
+    @Inject()
+    protected readonly container!: Container
+
+    @Inject()
+    protected readonly logging!: Logging
+
+    private cachedProxmoxApi?: Proxmox.Api
+
+    public async provisionMasterNode(): Promise<Node> {
+        // Look up the name of the master node from the settings
+        const pveMasterNode = await Setting.loadOneRequired('pveMasterNode')
+
+        // Look up the P5x storage pool from the settings
+        const pveStoragePool = await Setting.loadOneRequired('pveStoragePool')
+
+        // Look up the network bridge to which we will join the master node
+        const nodeNetworkBridge = await Setting.loadOneRequired('nodeNetworkBridge')
+
+        // Look up the primary IP pool range
+        const ipRange = await IpRange.getDefault()
+
+        // Look up the DNS search domain
+        const dnsDomain = await Setting.loadOneRequired('dnsDomain')
+
+        // Look up the root password for LXC containers
+        const rootPassword = await Setting.loadOneRequired('rootPassword')
+
+        // Get the Proxmox API
+        const proxmox = await this.getApi()
+
+        // Grab the master Proxmox node from the API
+        const masterNode = proxmox.nodes.$(pveMasterNode)
+
+        // Create a storage mount for the LXC template image
+        this.logging.info('Creating storage for container templates...')
+        const imageCacheStorage = await proxmox.storage.$post({
+            storage: 'p5x.image-cache',
+            type: 'dir',
+            content: ['vztmpl', 'iso'].join(','),
+            path: '/p5x-images',
+        })
+        const masterNodeStorage = masterNode.storage.$(imageCacheStorage.storage)
+
+        // Download the container template image
+        this.logging.info('Looking up LXC image for control node...')
+        const image = await this.images.getImages()
+            .then(x =>
+                x.where('os', '=', 'ubuntu')
+                    .where('version', '=', 'jammy')  // 22.04 LTS
+                    .where('format', '=', 'cloud')
+                    .where('arch', '=', 'amd64')
+                    .sortByDesc('timestamp')
+                    .first()
+            )
+
+        if ( !image ) {
+            throw new ErrorWithContext('Unable to find Debian Bookworm LXC template binary')
+        }
+
+        const imageUrl = this.images.getImageBinaryPath(image!)
+
+        // Upload the template image to the storage mount
+        const uploadTaskUPID = await masterNodeStorage['download-url'].$post({
+            content: 'vztmpl',
+            filename: 'p5x-base.tar.xz',
+            url: imageUrl.toRemote,
+        })
+
+        this.logging.info('Waiting for LXC image to upload to master PVE node...')
+        await this.waitForNodeTask(pveMasterNode, uploadTaskUPID)
+
+        // Get the next vmid to be used by the LXC container
+        const masterVMID = await proxmox.cluster.nextid.$get()
+        const masterIP = await ipRange.getNextAvailableOrFail()
+
+        // Create a new LXC container on that node for the control server
+        const sshPubKey = sshpk.parseKey(await Setting.loadOneRequired('sshPublicKey'))
+            .toString('ssh')
+
+        const masterTaskUPID = await masterNode.lxc.$post({
+            ostemplate: 'p5x.image-cache:vztmpl/p5x-base.tar.xz',
+            vmid: masterVMID,
+            cores: 1,
+            description: 'P5x Control Node',
+            hostname: 'p5x-control',
+            memory: 512,  // in MB
+            net0:
+                objectToKeyValue({
+                    name: 'eth0',
+                    bridge: nodeNetworkBridge,
+                    firewall: 1,
+                    gw: ipRange.gatewayIp,
+                    ip: `${masterIP}/${ipRange.subnet}`,
+                })
+                .map(x => `${x.key}=${x.value}`)
+                .join(','),
+            onboot: true,
+            password: rootPassword,
+            rootfs: `${pveStoragePool}:8`,  // 8 GiB
+            searchdomain: dnsDomain,
+            'ssh-public-keys': sshPubKey,
+            start: true,
+            storage: pveStoragePool,
+            tags: 'p5x',
+        })
+
+        this.logging.info('Waiting for PVE master node to be created...')
+        await this.waitForNodeTask(pveMasterNode, masterTaskUPID)
+
+        // Save the node
+        const node = this.container.makeNew<Node>(Node)
+        node.pveId = masterVMID
+        node.hostname = 'p5x-control'
+        node.assignedIp = masterIP
+        node.assignedSubnet = ipRange.subnet
+        node.isPermanent = true
+        node.isMaster = true
+        await node.save()
+
+        // Wait for the host to come online (this will be a PCTHost passed through PVE)
+        this.logging.info('Waiting for master node to come online...')
+        const proxyHost = await node.getHost()
+        await proxyHost.waitForAlive()
+
+        // Configure SSH on the instance
+        this.logging.info('Configuring SSH access to master node...')
+        await proxyHost.run(shellCommand('apt update'))
+        await proxyHost.run(shellCommand('apt install -y openssh-server curl'))
+        await proxyHost.run(shellCommand('systemctl enable --now ssh'))
+        await new Promise<void>(res => setTimeout(() => res(), 5000))
+
+        // Try to re-connect to the instance directly
+        this.logging.info('Attempting to re-connect to master node via SSH...')
+        const sshHost = await node.getHost()
+        if ( !(sshHost instanceof SSHHost) ) {
+            throw new ErrorWithContext('Failed to establish direct connection to master node', {
+                node,
+                proxyHost,
+                sshHost,
+            })
+        }
+
+        // Install base environment needed to run this control server
+        await sshHost.run(shellCommand('curl -fsSL https://deb.nodesource.com/setup_20.x | bash -'))
+        await sshHost.run(shellCommand('apt install git nodejs -y'))
+
+        return node
+    }
+
+    protected async waitForNodeTask(nodeName: string, taskUPID: string): Promise<void> {
+        const proxmox = await this.getApi()
+        const proxmoxNode = proxmox.nodes.$(nodeName)
+
+        while ( true ) {
+            const status = await proxmoxNode.tasks.$(taskUPID).status.$get()
+            if ( status.status === 'running' ) {
+                await new Promise(res => setTimeout(res, 1000))
+                continue
+            }
+
+            if ( status.exitstatus !== 'OK' ) {
+                throw new ErrorWithContext('PVE task did not succeed!', {
+                    nodeName,
+                    taskUPID,
+                    status,
+                })
+            }
+
+            break
+        }
+    }
+
+    public async getApi(): Promise<Proxmox.Api> {
+        // Have to force this to be a runtime ESM import since ts-node
+        // is re-writing it to a require(...), breaking the package.
+        const proxmoxApi = await unsafeESMImport('proxmox-api').then(x => x.proxmoxApi)
+
+        // If we have a cached version, just return that
+        if ( this.cachedProxmoxApi ) {
+            return this.cachedProxmoxApi
+        }
+
+        // Look up the master Proxmox node from the settings
+        const pveHost = await Setting.loadOneRequired('pveApiHost')
+
+        // Look up the Proxmox credentials from the settings
+        // const pveTokenId = await Setting.loadOneRequired('pveTokenId')
+        // const pveTokenSecret = await Setting.loadOneRequired('pveTokenSecret')
+        const pveRootPassword = await Setting.loadOneRequired('pveRootPassword')
+
+        // Initialize the Proxmox API
+        process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
+        this.cachedProxmoxApi = proxmoxApi({
+            host: pveHost,
+            user: 'root@pam',
+            password: pveRootPassword,
+            // tokenID: pveTokenId,
+            // tokenSecret: pveTokenSecret,
+        }) as Proxmox.Api
+
+        return this.cachedProxmoxApi
+    }
+
+}

src/app/support/hosts/ExecutionResult.ts

@@ -0,0 +1,36 @@
+export class ExecutionResult {
+    protected stdout: string[] = []
+    protected stderr: string[] = []
+    protected mixed: string[] = []
+    public exitCode?: number
+
+    out(data: any) {
+        this.stdout = this.stdout.concat(`${data}`.split('\n'))
+        this.mixed = this.mixed.concat(`${data}`.split('\n'))
+    }
+
+    error(data: any) {
+        this.stderr = this.stderr.concat(`${data}`.split('\n'))
+        this.mixed = this.mixed.concat(`${data}`.split('\n'))
+    }
+
+    exit(code: number) {
+        this.exitCode = code
+    }
+
+    public wasSuccessful(): boolean {
+        return this.exitCode === 0
+    }
+
+    public get combinedOutput(): string[] {
+        return this.mixed.filter(Boolean)
+    }
+
+    public get standardOutput(): string[] {
+        return this.stdout.filter(Boolean)
+    }
+
+    public get errorOutput(): string[] {
+        return this.stderr.filter(Boolean)
+    }
+}

src/app/support/hosts/Host.ts

@@ -0,0 +1,203 @@
+import {
+    Awaitable,
+    AwareOfContainerLifecycle, ErrorWithContext,
+    Filesystem,
+    infer,
+    objectToKeyValue,
+    UniversalPath,
+    uuid4,
+} from '@extollo/lib'
+import {CommandPalette, CommandPaletteCommands, OwnershipSpec, shellCommand, ShellCommand} from './types'
+import {ExecutionResult} from './ExecutionResult'
+import {CommandError} from './errors'
+import {SystemMetrics} from './SystemMetrics'
+
+export class StandardCommandPalette implements CommandPalette {
+    changeDirectory = shellCommand(`cd "%%PATH%%"`)
+    cpuPercentage = shellCommand(`grep 'cpu ' /proc/stat | awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage}'`)
+    fileDirectoryDelete = shellCommand(`rm -rf "%%RESOURCE%%"`)
+    fileDirectoryOwnershipFetch = shellCommand(`stat -c "%U:%G" "%%PATH%%"`)
+    fileDirectoryOwnershipSetFlat = shellCommand(`chown %%OWNERS%% "%%PATH%%"`)
+    fileDirectoryOwnershipSetRecursive = shellCommand(`chown -R %%OWNERS%% "%%PATH%%"`)
+    fileDirectoryPermissionFetch = shellCommand(`stat -c %a "%%PATH%%"`)
+    fileDirectoryPermissionSetFlat = shellCommand(`chmod %%LEVEL%% "%%PATH%%"`)
+    fileDirectoryPermissionSetRecursive = shellCommand(`chmod -R %%LEVEL%% "%%PATH%%"`)
+    listMountPoints = shellCommand(`df -hl | grep '/' | awk '{print $6}`)
+    mountPointPercentage = shellCommand(`df -hl | grep -w '%%MOUNTPOINT%%$' | awk '{print $5}'`)
+    ramPercentage = shellCommand(`free | grep Mem | awk '{print $3/$2 * 100.0}'`)
+    reboot = shellCommand(`reboot`)
+    resolvePath = shellCommand(`readlink -f "%%PATH%%"`)
+    tempFile = shellCommand(`mktemp`)
+    tempPath = shellCommand(`mktemp -d`)
+    echo = shellCommand(`echo "%%OUTPUT%%"`)
+
+    public format(command: keyof CommandPaletteCommands, substitutions: Record<string, any>): ShellCommand {
+        let formatted = `${this[command]}`
+
+        objectToKeyValue(substitutions)
+            .map(item =>
+                formatted = formatted.replaceAll(`%%${item.key.toUpperCase()}%%`, `${item.value}`))
+
+        return shellCommand(formatted)
+    }
+}
+
+export abstract class Host implements AwareOfContainerLifecycle {
+    awareOfContainerLifecycle: true = true
+
+    public abstract getFilesystem(): Awaitable<Filesystem>
+
+    public abstract execute(command: ShellCommand): Awaitable<ExecutionResult>
+
+    close(): Awaitable<void> {}
+
+    protected getCommandPalette(): CommandPalette {
+        return new StandardCommandPalette()
+    }
+
+    onContainerDestroy() {
+        this.close()
+    }
+
+    onContainerRelease() {
+        this.close()
+    }
+
+    /** Runs the provided command and throws if it failed. */
+    public async run(command: ShellCommand): Promise<ExecutionResult> {
+        const result = await this.execute(command)
+        if ( !result.wasSuccessful() ) {
+            throw CommandError.make(command, result)
+        }
+        return result
+    }
+
+    /** Runs the provided command, throws if it failed, and returns the `infer(...)` result of the first line otherwise. */
+    public async runLineResult(command: ShellCommand): Promise<ReturnType<typeof infer>> {
+        const result = await this.execute(command)
+        if ( !result.wasSuccessful() || result.standardOutput.length < 1 ) {
+            throw CommandError.make(command, result)
+        }
+        return infer(result.standardOutput[0].trim())
+    }
+
+    /** Try to ping the host and verify that it is online & accessible. */
+    public async isAlive(): Promise<boolean> {
+        const output = uuid4()
+        const cmd = this.getCommandPalette().format('echo', { output })
+        try {
+            const result = await this.execute(cmd)
+            return result.wasSuccessful() && result.combinedOutput.length > 0 && result.combinedOutput[0] === output
+        } catch (e) {
+            return false
+        }
+    }
+
+    public async waitForAlive(maxTries: number = 30): Promise<void> {
+        let tries = maxTries
+        while ( tries > 0 ) {
+            if ( await this.isAlive() ) {
+                return
+            }
+
+            tries -= 1
+            await new Promise<void>(res => setTimeout(() => res(), 2000))
+        }
+
+        throw new ErrorWithContext('Host did not come online before maxTries exceeded', {
+            host: this,
+            maxTries,
+        })
+    }
+
+    /** Get the path to a temporary directory on the host. */
+    public async getTempDirectory(): Promise<UniversalPath> {
+        const cmd = this.getCommandPalette().format('tempPath', {})
+        const result = await this.runLineResult(cmd)
+        return new UniversalPath(`${result}`, await this.getFilesystem())
+    }
+
+    /** Get the path to a temporary file on the host. */
+    public async getTempFile(): Promise<UniversalPath> {
+        const cmd = this.getCommandPalette().format('tempFile', {})
+        const result = await this.runLineResult(cmd)
+        return new UniversalPath(`${result}`, await this.getFilesystem())
+    }
+
+    /** Get a list of filesystem mount points on the host. */
+    public async getMountPoints(): Promise<string[]> {
+        const result = await this.run(this.getCommandPalette().format('listMountPoints', {}))
+        return result.standardOutput
+    }
+
+    /** Load standard metrics about the host. */
+    public async metrics(): Promise<SystemMetrics> {
+        const metric = new SystemMetrics()
+
+        const palette = this.getCommandPalette()
+        const cpuCmd = palette.format('cpuPercentage', {})
+        const cpuResult = await this.runLineResult(cpuCmd)
+        if ( typeof cpuResult === 'number' ) {
+            metric.cpuPercent = cpuResult
+        }
+
+        const ramCmd = palette.format('ramPercentage', {})
+        const ramResult = await this.runLineResult(ramCmd)
+        if ( typeof ramResult === 'number' ) {
+            metric.ramPercent = ramResult
+        }
+
+        const mountPoints = await this.getMountPoints()
+        await Promise.all(mountPoints.map(async mountPoint => {
+            const cmd = palette.format('mountPointPercentage', { mountPoint })
+            const result = await this.run(cmd)
+            const utilization = Number(result.standardOutput[0].replace('%', '')) / 100
+            metric.setMount(mountPoint, utilization)
+        }))
+
+        return metric
+    }
+
+    /** Get the user/group ownership of the given resource. */
+    public async getOwnership(path: UniversalPath): Promise<OwnershipSpec> {
+        const cmd = this.getCommandPalette().format('fileDirectoryOwnershipFetch', { path: path.toLocal })
+        const result = `${await this.runLineResult(cmd)}`
+        const parts = result.split(':')
+        return {user: parts[0], group: parts[1]}
+    }
+
+    /** Set the user/group ownership of the given resource. */
+    public setOwnership(path: UniversalPath, user: string, group: string, recursive: boolean = false): Awaitable<ExecutionResult> {
+        const cmd = this.getCommandPalette().format(
+            recursive ? 'fileDirectoryOwnershipSetRecursive' : 'fileDirectoryOwnershipSetFlat',
+            {
+                path: path.toLocal,
+                owners: `${user}:${group}`,
+            }
+        )
+        return this.run(cmd)
+    }
+
+    /** Retrieve the filesystem permissions for the given resource. */
+    public async getPermissions(path: UniversalPath): Promise<string> {
+        const cmd = this.getCommandPalette().format('fileDirectoryPermissionFetch', { path: path.toLocal })
+        return `${await this.runLineResult(cmd)}`
+    }
+
+    /** Set the filesystem permissions for the given resource. */
+    public setPermissions(path: UniversalPath, level: string, recursive: boolean = false): Awaitable<ExecutionResult> {
+        const cmd = this.getCommandPalette().format(
+            recursive ? 'fileDirectoryPermissionSetRecursive' : 'fileDirectoryPermissionSetFlat',
+            {
+                path: path.toLocal,
+                level,
+            }
+        )
+        return this.run(cmd)
+    }
+
+    /** Reboot the host. */
+    public reboot(): Awaitable<ExecutionResult> {
+        return this.run(this.getCommandPalette().format('reboot', {}))
+    }
+}

src/app/support/hosts/LocalHost.ts

@@ -0,0 +1,25 @@
+import {Host} from './Host'
+import {ShellCommand} from './types'
+import {Application, Awaitable, Filesystem, LocalFilesystem} from '@extollo/lib'
+import {ExecutionResult} from './ExecutionResult'
+import * as childProcess from 'child_process'
+
+export class LocalHost extends Host {
+    async execute(command: ShellCommand): Promise<ExecutionResult> {
+        const result = new ExecutionResult()
+        return new Promise(res => {
+            childProcess.exec(command, (error, stdout, stderr) => {
+                result.exit(error?.code || 0)
+                result.out(stdout)
+                result.error(stderr)
+                res(result)
+            })
+        })
+    }
+
+    getFilesystem(): Awaitable<Filesystem> {
+        return Application.getContainer().makeNew<LocalFilesystem>(LocalFilesystem, {
+            baseDir: '/'
+        })
+    }
+}

src/app/support/hosts/PCTHost.ts

@@ -0,0 +1,24 @@
+import {SSHHost} from './SSHHost'
+import * as ssh2 from 'ssh2'
+import {shellCommand, ShellCommand} from './types'
+import {ExecutionResult} from './ExecutionResult'
+import {ErrorWithContext, Filesystem} from '@extollo/lib'
+
+export class PCTHost extends SSHHost {
+    constructor(
+        config: ssh2.ConnectConfig,
+        protected readonly vmid: number,
+    ) {
+        super(config)
+    }
+
+    async execute(command: ShellCommand): Promise<ExecutionResult> {
+        return super.execute(shellCommand('pct exec', `${this.vmid}`, '--', command))
+    }
+
+    getFilesystem(): Promise<Filesystem>{
+        throw new ErrorWithContext('Accessing Filesystem of PCTHost is not supported.', {
+            vmid: this.vmid,
+        })
+    }
+}

src/app/support/hosts/SSHHost.ts

@@ -0,0 +1,77 @@
+import {Host} from './Host'
+import {Application, Awaitable, Filesystem, SSHFilesystem} from '@extollo/lib'
+import {ExecutionResult} from './ExecutionResult'
+import {ShellCommand} from './types'
+import * as ssh2 from 'ssh2'
+
+export class SSHHost extends Host {
+    private sshClient?: ssh2.Client
+    private filesystem?: Filesystem
+
+    constructor(
+        protected readonly config: ssh2.ConnectConfig,
+    ) {
+        super()
+    }
+
+    async close() {
+        this.sshClient?.end()
+        this.sshClient?.destroy()
+        this.sshClient = undefined
+
+        this.filesystem?.close()
+        this.filesystem = undefined
+    }
+
+    async execute(command: ShellCommand): Promise<ExecutionResult>{
+        const client = await this.getSSH()
+        const result = new ExecutionResult()
+        return new Promise<ExecutionResult>((res, rej) => {
+            client.exec(command, (err, stream) => {
+                if ( err ) {
+                    return rej(err)
+                }
+
+                stream
+                    .on('close', (code: number) => {
+                        result.exit(code)
+                        res(result)
+                    })
+                    .on('data', (data: any) => {
+                        result.out(data)
+                    })
+                    .stderr.on('data', (data: any) => {
+                        result.error(data)
+                    })
+            })
+        })
+    }
+
+    getFilesystem(): Awaitable<Filesystem> {
+        if ( !this.filesystem ) {
+            this.filesystem = Application.getContainer().makeNew<SSHFilesystem>(SSHFilesystem, {
+                ssh: this.config,
+                baseDir: '/',
+            })
+        }
+
+        return this.filesystem
+    }
+
+    protected async getSSH(): Promise<ssh2.Client> {
+        if ( this.sshClient ) {
+            return this.sshClient
+        }
+
+        return new Promise((res, rej) => {
+            const client = new ssh2.Client()
+
+            client.on('ready', () => {
+                this.sshClient = client
+                res(client)
+            }).connect(this.config)
+
+            client.on('error', rej)
+        })
+    }
+}

src/app/support/hosts/SystemMetrics.ts

@@ -0,0 +1,15 @@
+import {Maybe} from '@extollo/lib'
+
+export class SystemMetrics {
+    public cpuPercent: number = 0
+    public ramPercent: number = 0
+    protected mountPercents: Record<string, number> = {}
+
+    public setMount(point: string, percent: number) {
+        this.mountPercents[point] = percent
+    }
+
+    public mountPercent(point: string): Maybe<number> {
+        return this.mountPercents[point]
+    }
+}

src/app/support/hosts/errors.ts

@@ -0,0 +1,51 @@
+import {ErrorWithContext} from '@extollo/lib'
+import {ShellCommand} from './types'
+import {ExecutionResult} from './ExecutionResult'
+
+export class CommandError extends ErrorWithContext {
+    public static make(command: ShellCommand, result: ExecutionResult): CommandError {
+        const combinedOutput = result.combinedOutput.join(' ').toLowerCase()
+
+        const accessDeniedPhrases = [
+            'not permitted',
+            'access denied',
+            'insufficient permission',
+            'permission denied'
+        ]
+
+        for ( const phrase of accessDeniedPhrases ) {
+            if ( combinedOutput.includes(phrase) ) return new PermissionDeniedError(command, result)
+        }
+
+        const notFoundPhrases = [
+            'command not found',
+        ]
+
+        for ( const phrase of notFoundPhrases ) {
+            if ( combinedOutput.includes(phrase) ) return new CommandNotFoundError(command, result)
+        }
+
+        const resourceNotFoundPhrases = [
+            'no such file',
+            'no such directory',
+            'no such file or directory',
+            'no such directory or file',
+        ]
+
+        for ( const phrase of resourceNotFoundPhrases ) {
+            if ( combinedOutput.includes(phrase) ) return new FilesystemResourceNotFoundError(command, result)
+        }
+
+        return new CommandError(command, result)
+    }
+
+    constructor(command: ShellCommand, result: ExecutionResult) {
+        super(`Unable to execute command: ${command}`, { command, result })
+    }
+}
+
+export class PermissionDeniedError extends CommandError {}
+
+export class CommandNotFoundError extends CommandError {}
+
+export class FilesystemResourceNotFoundError extends CommandError {}

src/app/support/hosts/index.ts

@@ -0,0 +1,7 @@
+export * from './types'
+export * from './errors'
+export * from './ExecutionResult'
+export * from './SystemMetrics'
+export * from './Host'
+export * from './LocalHost'
+export * from './SSHHost'

src/app/support/hosts/types.ts

@@ -0,0 +1,34 @@
+import {TypeTag} from '@extollo/lib'
+
+export type ShellCommand = TypeTag<'p5x.shell-command'> & string
+
+export const shellCommand = (...parts: string[]): ShellCommand => parts.join(' ') as ShellCommand
+
+export interface CommandPaletteCommands {
+    tempPath: ShellCommand
+    tempFile: ShellCommand
+    cpuPercentage: ShellCommand
+    ramPercentage: ShellCommand
+    mountPointPercentage: ShellCommand
+    listMountPoints: ShellCommand
+    fileDirectoryDelete: ShellCommand
+    resolvePath: ShellCommand
+    reboot: ShellCommand
+    changeDirectory: ShellCommand
+    fileDirectoryPermissionFetch: ShellCommand
+    fileDirectoryPermissionSetFlat: ShellCommand
+    fileDirectoryPermissionSetRecursive: ShellCommand
+    fileDirectoryOwnershipFetch: ShellCommand
+    fileDirectoryOwnershipSetFlat: ShellCommand
+    fileDirectoryOwnershipSetRecursive: ShellCommand
+    echo: ShellCommand
+}
+
+export type CommandPalette = CommandPaletteCommands & {
+    format(command: keyof CommandPaletteCommands, substitutions: Record<string, any>): ShellCommand
+}
+
+export interface OwnershipSpec {
+    user: string
+    group: string
+}

src/app/types.ts

@@ -0,0 +1,31 @@
+import {TypeTag, JSONState, isJSON} from '@extollo/lib'
+import * as IPCIDR from 'ip-cidr'
+
+export type IpAddress = TypeTag<'p5x.ip.address'> & string
+export const isIpAddress = (what: unknown): what is IpAddress =>
+    typeof what === 'string' && IPCIDR.isValidAddress(what)
+
+export type Subnet = TypeTag<'p5x.ip.subnet'> & number
+export const isSubnet = (what: unknown): what is Subnet =>
+    typeof what === 'number' && IPCIDR.isValidCIDR(`1.1.1.1/${what}`)
+
+export type JSONString = TypeTag<'p5x.json'> & string
+export const isJSONString = (what: unknown): what is JSONString =>
+    typeof what === 'string' && isJSON(what)
+
+export type Settings = Record<string, JSONState> & {
+    pveMasterNode: string
+    pveApiHost: string
+    pveStoragePool: string
+    // pveTokenId: string
+    // pveTokenSecret: string
+    pveRootPassword: string
+    lxcTemplateFileName: string
+    rootPassword: string
+    sshPublicKey: string
+    sshPrivateKey: string
+    dnsDomain: string
+    nodeNetworkBridge: string
+    nodeCpus: number
+    nodeRamMib: number
+}

src/cli.ts

@@ -12,4 +12,5 @@ globalRegistry.run(async () => {
      */
     const app = cli()
     await app.run()
+    await app.destroy()
 })

src/index.ts

@@ -9,5 +9,7 @@ globalRegistry.run(async () => {
      * ties your entire application together. The app container manages services
      * and lifecycle.
      */
-    await app().run()
+    const theApp = app()
+    await theApp.run()
+    await theApp.destroy()
 })