Create initial standup logic to bootstrap control CT & install Node.js

This commit is contained in:
Garrett Mills 2023-07-16 17:48:03 -05:00
parent 0866528cbe
commit 3a6c24b603
27 changed files with 2241 additions and 688 deletions

2
.gitignore vendored
View File

@ -193,3 +193,5 @@ dist
/exbuild
*.sqlite
notes.md
src/dev_tools.ts

View File

@ -9,17 +9,26 @@
},
"dependencies": {
"@atao60/fse-cli": "^0.1.7",
"@extollo/lib": "^0.14.10",
"@extollo/lib": "^0.14.13",
"@types/node": "^18.15.11",
"@types/ssh2": "^1.11.13",
"@types/sshpk": "^1.17.1",
"@types/ws": "^8.5.5",
"copyfiles": "^2.4.1",
"engine.io-client": "^6.5.1",
"feed": "^4.2.2",
"gotify": "^1.1.0",
"ip-cidr": "^3.1.0",
"proxmox-api": "^1.0.0",
"rimraf": "^3.0.2",
"ssh2": "^1.14.0",
"sshpk": "^1.17.0",
"ts-expose-internals": "^4.5.4",
"ts-node": "^10.9.0",
"ts-patch": "^2.0.1",
"ts-to-zod": "^1.8.0",
"typescript": "^4.3.2",
"ws": "^8.13.0",
"zod": "^3.11.6"
},
"scripts": {

File diff suppressed because it is too large Load Diff

View File

@ -1,9 +1,14 @@
import {ORMUser, Singleton, Unit} from '@extollo/lib'
import {ORMUser, Singleton, Unit, CommandLine, Inject} from '@extollo/lib'
import {User} from './models/User.model'
import {StandupDirective} from './cli/directive/StandupDirective'
@Singleton()
export class AppUnit extends Unit {
@Inject()
protected readonly cli!: CommandLine
async up(): Promise<void> {
this.container().registerStaticOverride(ORMUser, User)
this.cli.registerDirective(StandupDirective)
}
}

View File

@ -0,0 +1,135 @@
import {Directive, Inject, Injectable, OptionDefinition, uuid4} from '@extollo/lib'
import { IpRange } from "../../models/IpRange.model";
import { Setting } from "../../models/Setting.model";
import { Provisioner } from "../../services/Provisioner.service";
import { IpAddress, isIpAddress, isSubnet, Subnet } from "../../types";
import * as crypto from 'crypto'
@Injectable()
export class StandupDirective extends Directive {
@Inject()
protected readonly provisioner!: Provisioner
getKeywords(): string | string[] {
return 'standup'
}
getDescription(): string {
return 'perform initial configuration and setup for the control server'
}
getOptions(): OptionDefinition[] {
return [
'--pve-master-node {name} | the name of the master node in the PVE cluster',
'--pve-api-host {host} | the hostname or IP of a PVE cluster node',
// '--pve-token-id {id} | the fully-qualified ID of a PVE API token',
// '--pve-token-secret {secret} | secret for the PVE API token',
'--pve-root-password {password} | password for the root@pam PVE user',
'--ip-pool-start {start} | the starting address in the P5X IP pool',
'--ip-pool-end {end} | the final address in the P5X IP pool',
'--ip-pool-subnet {subnet} | the numeric subnet of the P5X IP pool (e.g. 24)',
'--ip-pool-gateway {gateway} | the address of the gateway for the P5X IP pool',
'--storage-pool {name} | the name of the PVE shared storage pool',
'--dns-domain {name} | the DNS domain name used for K8s nodes',
'--node-network-bridge {name} | the name of the PVE network bridge to which nodes will join',
'--node-cpu-cores {cores} | the number of cores to provision each K8s node with',
'--node-ram-mib {ram} | the amount of ram (in MiB) to provision each K8s node with',
]
}
public async handle(argv: string[]): Promise<void> {
// Configure the settings:
await Setting.set('pveMasterNode', `${this.getOptionOrFail('pve-master-node')}`)
await Setting.set('pveApiHost', `${this.getOptionOrFail('pve-api-host')}`)
// await Setting.set('pveTokenId', `${this.getOptionOrFail('pve-token-id')}`)
// await Setting.set('pveTokenSecret', `${this.getOptionOrFail('pve-token-secret')}`)
await Setting.set('pveRootPassword', `${this.getOptionOrFail('pve-root-password')}`)
await Setting.set('pveStoragePool', `${this.getOptionOrFail('storage-pool')}`)
await Setting.set('dnsDomain', `${this.getOptionOrFail('dns-domain')}`)
await Setting.set('nodeNetworkBridge', `${this.getOptionOrFail('node-network-bridge')}`)
await Setting.set('nodeCpus', this.getIntOptionOrFail('node-cpu-cores'))
await Setting.set('nodeRamMib', this.getIntOptionOrFail('node-ram-mib'))
await Setting.set('rootPassword', uuid4())
const [sshPublicKey, sshPrivateKey] = await this.getSSHKeyPair()
await Setting.set('sshPublicKey', sshPublicKey)
await Setting.set('sshPrivateKey', sshPrivateKey)
// Store the IP address pool:
const ipRange = this.container().makeNew<IpRange>(IpRange)
ipRange.name = 'Default address pool'
ipRange.startIp = this.getIpOptionOrFail('ip-pool-start')
ipRange.endIp = this.getIpOptionOrFail('ip-pool-end')
ipRange.subnet = this.getSubnetOptionOrFail('ip-pool-subnet')
ipRange.gatewayIp = this.getIpOptionOrFail('ip-pool-gateway')
ipRange.save()
// Provision the master node:
const masterNode = await this.provisioner.provisionMasterNode()
}
protected getIntOptionOrFail(option: string): number {
const val = parseInt(this.option(option, NaN), 10)
if ( isNaN(val) ) {
this.error(`Missing or malformed required integer option: --${option}`)
process.exit(1)
}
return val
}
protected getSubnetOptionOrFail(option: string): Subnet {
const val = parseInt(this.option(option, NaN), 10)
if ( !isSubnet(val) ) {
this.error(`Missing or malformed required numeric subnet: --${option}`)
process.exit(1)
}
return val
}
protected getIpOptionOrFail(option: string): IpAddress {
const val = `${this.option(option, '')}`
if ( !isIpAddress(val) ) {
this.error(`Missing or malformed required IP address: --${option}`)
process.exit(1)
}
return val
}
protected getOptionOrFail(option: string): unknown {
const val = this.option(option, undefined)
if ( typeof val === 'undefined' ) {
this.error(`Missing required option: --${option}`)
process.exit(1)
}
return val
}
protected async getSSHKeyPair(): Promise<[string, string]> {
return new Promise<[string, string]>((res, rej) => {
crypto.generateKeyPair(
'rsa',
{
modulusLength: 4096,
publicKeyEncoding: {
type: 'spki',
format: 'pem',
},
privateKeyEncoding: {
type: 'pkcs8',
format: 'pem',
cipher: 'aes-256-cbc',
passphrase: '',
},
},
(err, pubkey, privkey) => {
if ( err ) {
return rej(err)
}
res([pubkey, privkey])
},
)
})
}
}

View File

@ -0,0 +1,48 @@
import {Injectable, Migration, Inject, DatabaseService, FieldType} from '@extollo/lib'
/**
* CreateP5xCredentialsTableMigration
* ----------------------------------
* Create a table to store the PVE user credentials.
*/
@Injectable()
export default class CreateP5xCredentialsTableMigration extends Migration {
@Inject()
protected readonly db!: DatabaseService
/**
* Apply the migration.
*/
async up(): Promise<void> {
const schema = this.db.get().schema()
const table = await schema.table('p5x_credentials')
table.primaryKey('id')
table.column('pve_username')
.type(FieldType.varchar)
.required()
table.column('pve_credential')
.type(FieldType.text)
.required()
table.column('is_api_token')
.type(FieldType.bool)
.required()
await schema.commit(table)
}
/**
* Undo the migration.
*/
async down(): Promise<void> {
const schema = this.db.get().schema()
const table = await schema.table('p5x_credentials')
table.dropIfExists()
await schema.commit(table)
}
}

View File

@ -0,0 +1,56 @@
import {Injectable, Migration, Inject, DatabaseService, FieldType} from '@extollo/lib'
/**
* CreateP5xIpRangesTableMigration
* ----------------------------------
* Put some description here.
*/
@Injectable()
export default class CreateP5xIpRangesTableMigration extends Migration {
@Inject()
protected readonly db!: DatabaseService
/**
* Apply the migration.
*/
async up(): Promise<void> {
const schema = this.db.get().schema()
const table = await schema.table('p5x_ip_ranges')
table.primaryKey('id')
table.column('name')
.type(FieldType.varchar)
.required()
table.column('start_ip')
.type(FieldType.varchar)
.required()
table.column('end_ip')
.type(FieldType.varchar)
.required()
table.column('subnet')
.type(FieldType.int)
.required()
table.column('gateway_ip')
.type(FieldType.varchar)
.required()
await schema.commit(table)
}
/**
* Undo the migration.
*/
async down(): Promise<void> {
const schema = this.db.get().schema()
const table = await schema.table('p5x_ip_ranges')
table.dropIfExists()
await schema.commit(table)
}
}

View File

@ -0,0 +1,62 @@
import {Injectable, Migration, Inject, DatabaseService, FieldType} from '@extollo/lib'
/**
* CreateP5xNodesTableMigration
* ----------------------------------
* Create a table to track the virtual K8s containers under our control.
*/
@Injectable()
export default class CreateP5xNodesTableMigration extends Migration {
@Inject()
protected readonly db!: DatabaseService
/**
* Apply the migration.
*/
async up(): Promise<void> {
const schema = this.db.get().schema()
const table = await schema.table('p5x_nodes')
table.primaryKey('id')
table.column('pve_id')
.type(FieldType.int)
.required()
table.column('hostname')
.type(FieldType.varchar)
.required()
table.column('assigned_ip')
.type(FieldType.varchar)
.required()
table.column('assigned_subnet')
.type(FieldType.int)
.required()
table.column('is_permanent')
.type(FieldType.bool)
.default(false)
.required()
table.column('is_master')
.type(FieldType.bool)
.default(false)
.required()
await schema.commit(table)
}
/**
* Undo the migration.
*/
async down(): Promise<void> {
const schema = this.db.get().schema()
const table = await schema.table('p5x_nodes')
table.dropIfExists()
await schema.commit(table)
}
}

View File

@ -0,0 +1,44 @@
import {Injectable, Migration, Inject, DatabaseService, FieldType} from '@extollo/lib'
/**
* CreateSettingsTableMigration
* ----------------------------------
* Create a table to store random one-off settings.
*/
@Injectable()
export default class CreateSettingsTableMigration extends Migration {
@Inject()
protected readonly db!: DatabaseService
/**
* Apply the migration.
*/
async up(): Promise<void> {
const schema = this.db.get().schema()
const table = await schema.table('p5x_settings')
table.primaryKey('id')
table.column('setting_key')
.type(FieldType.varchar)
.required()
table.column('setting_json')
.type(FieldType.text)
.required()
await schema.commit(table)
}
/**
* Undo the migration.
*/
async down(): Promise<void> {
const schema = this.db.get().schema()
const table = await schema.table('p5x_settings')
table.dropIfExists()
await schema.commit(table)
}
}

View File

@ -0,0 +1,24 @@
import {Injectable, Model, Field, FieldType} from '@extollo/lib'
/**
* Credential Model
* -----------------------------------
* A username and {password|api token} for the PVE cluster.
*/
@Injectable()
export class Credential extends Model<Credential> {
protected static table = 'p5x_credentials'
protected static key = 'id'
@Field(FieldType.serial)
public id?: number
@Field(FieldType.varchar, 'pve_username')
public username!: string
@Field(FieldType.text, 'pve_credential')
public credential!: string
@Field(FieldType.bool, 'is_api_token')
public isApiToken!: boolean
}

View File

@ -0,0 +1,66 @@
import {Injectable, Model, Field, FieldType, TypeTag, Maybe, collect, ErrorWithContext} from '@extollo/lib'
import * as IPCIDR from 'ip-cidr'
import {IpAddress, Subnet} from '../types'
import {Node} from './Node.model'
/**
* IpRange Model
* -----------------------------------
* Represents an available IP address pool from which we can allocate node addresses.
*/
@Injectable()
export class IpRange extends Model<IpRange> {
protected static table = 'p5x_ip_ranges'
protected static key = 'id'
public static async getDefault(): Promise<IpRange> {
const ipRange = await this.query<IpRange>().first()
if ( !ipRange ) {
throw new Error('Unable to find default IP range!')
}
return ipRange
}
@Field(FieldType.serial)
public id?: number
@Field(FieldType.varchar)
public name!: string
@Field(FieldType.varchar, 'start_ip')
public startIp!: IpAddress
@Field(FieldType.varchar, 'end_ip')
public endIp!: IpAddress
@Field(FieldType.int)
public subnet!: Subnet
@Field(FieldType.varchar, 'gateway_ip')
public gatewayIp!: IpAddress
/** Get the next IP address in this range that has not already been allocated to a node. */
public async getNextAvailable(): Promise<Maybe<IpAddress>> {
// Generate the address pool
const cidr = new IPCIDR(`${this.gatewayIp}/${this.subnet}`)
const possibleIps = collect(cidr.toArray({ from: this.startIp, to: this.endIp }) as IpAddress[])
.filterOut(x => x.endsWith('.0'))
// Look up already-allocated addresses
const allocatedIps = await Node.query<Node>().get().pluck('assignedIp')
// Find the first available address
return possibleIps.firstWhere(ip => !allocatedIps.includes(ip))
}
/** Like `getNextAvailable(...)`, but it will throw if the range is exhausted */
public async getNextAvailableOrFail(): Promise<IpAddress> {
const ip = await this.getNextAvailable()
if ( !ip ) {
throw new ErrorWithContext('Unable to get next available IP from range: is range exhausted?', {
range: this.toObject(),
})
}
return ip
}
}

View File

@ -0,0 +1,61 @@
import {Injectable, Model, Field, FieldType} from '@extollo/lib'
import {IpAddress, Subnet} from '../types'
import {Host, SSHHost} from '../support/hosts'
import * as ssh2 from 'ssh2'
import * as sshpk from 'sshpk'
import {Setting} from './Setting.model'
import {PCTHost} from '../support/hosts/PCTHost'
/**
* Node Model
* -----------------------------------
* Represents a single LXC container running a K8s worker node.
*/
@Injectable()
export class Node extends Model<Node> {
protected static table = 'p5x_nodes'
protected static key = 'node_id'
@Field(FieldType.serial)
public id?: number
@Field(FieldType.varchar, 'pve_id')
public pveId!: number
@Field(FieldType.varchar)
public hostname!: string
@Field(FieldType.varchar, 'assigned_ip')
public assignedIp!: IpAddress
@Field(FieldType.int, 'assigned_subnet')
public assignedSubnet!: Subnet
@Field(FieldType.bool, 'is_permanent')
public isPermanent: boolean = false
@Field(FieldType.bool, 'is_master')
public isMaster: boolean = false
public async getHost(): Promise<Host> {
// Try to make a direct SSH connection to the container
const privKey = await Setting.loadOneRequired('sshPrivateKey')
const formattedPrivKey = sshpk.parsePrivateKey(privKey, 'pem', {passphrase: ''}).toString('ssh')
const directHost = this.container().makeNew<Host>(SSHHost, {
host: this.assignedIp,
username: 'root',
privateKey: formattedPrivKey,
} as ssh2.ConnectConfig)
if ( await directHost.isAlive() ) {
return directHost
}
// Otherwise, fall back to the PCTHost proxy
return this.container().makeNew<Host>(PCTHost, {
host: await Setting.loadOneRequired('pveApiHost'),
username: 'root',
password: await Setting.loadOneRequired('pveRootPassword'),
}, this.pveId)
}
}

View File

@ -0,0 +1,80 @@
import {Injectable, Model, Field, FieldType, Collection, ErrorWithContext, make, Application} from '@extollo/lib'
import { JSONString, Settings } from '../types';
export class MissingRequiredSettingError extends ErrorWithContext {
constructor(key: string) {
super(`Missing required setting: ${key}`, { key })
}
}
/**
* Setting Model
* -----------------------------------
* Helper for loading various one-off settings.
*/
@Injectable()
export class Setting<Key extends keyof Settings> extends Model<Setting<Key>> {
protected static table = 'p5x_settings'
protected static key = 'id'
public static load<Key extends keyof Settings>(key: Key): Promise<Collection<Settings[Key]>> {
return Setting.query<Setting<Key>>()
.where('setting_key', '=', key)
.get()
.map(x => x.value)
}
public static async loadRequired<Key extends keyof Settings>(key: Key): Promise<Collection<Settings[Key]>> {
const load = await this.load(key)
if ( load.isEmpty() ) {
throw new MissingRequiredSettingError(key)
}
return load
}
public static loadOneRequired<Key extends keyof Settings>(key: Key): Promise<Settings[Key]> {
return this.loadRequired(key).then(x => x.first()!)
}
public static async set<Key extends keyof Settings>(key: Key, value: Settings[Key]): Promise<void> {
// Delete any existing records w/ this key
await Setting.query().where('settingKey', '=', key).delete()
// Insert a new one
const setting = Application.getContainer().makeNew<Setting<Key>>(Setting);
setting.settingKey = key
setting.value = value
await setting.save()
}
public static async add<Key extends keyof Settings>(key: Key, value: Settings[Key]): Promise<void> {
// Delete any existing records w/ this key and value
await Setting.query()
.where('settingKey', '=', key)
.where('settingValue', '=', JSON.stringify(value))
.delete()
// Insert a new one
const setting = new Setting<Key>;
setting.settingKey = key
setting.value = value
await setting.save()
}
@Field(FieldType.serial)
protected id?: number
@Field(FieldType.varchar, 'setting_key')
protected settingKey!: Key
@Field(FieldType.text, 'setting_json')
protected settingJson!: JSONString
public get value(): Settings[Key] {
return JSON.parse(this.settingJson)
}
public set value(v: Settings[Key]) {
this.settingJson = JSON.stringify(v) as JSONString
}
}

View File

@ -0,0 +1,38 @@
import {appPath, AsyncCollection, collect, Collection, Singleton, UniversalPath, universalPath} from '@extollo/lib'
export interface LXCImage {
os: string
version: string
arch: 'amd64' | 'arm64' | 'i386' | string
format: 'cloud' | 'default'
timestamp: string
endpoint: string
}
@Singleton()
export class Images {
private imageServer = 'https://us.lxd.images.canonical.com'
private listEndpoint = 'meta/1.0/index-system'
private imageBinaryName = 'rootfs.tar.xz'
public async getImages(): Promise<Collection<LXCImage>> {
const url = universalPath(this.imageServer, this.listEndpoint)
const content = await fetch(url.toRemote).then(x => x.text())
return collect(content.trim().split('\n'))
.map(row => {
const [os, version, arch, format, timestamp, endpoint] = row.split(';')
return {
os,
version,
arch,
format,
timestamp,
endpoint,
} as LXCImage
})
}
public getImageBinaryPath(image: LXCImage): UniversalPath {
return universalPath(this.imageServer, image.endpoint, this.imageBinaryName)
}
}

View File

@ -0,0 +1,228 @@
import {
Singleton,
Inject,
ErrorWithContext,
objectToKeyValue,
fetch,
Container,
Logging,
universalPath,
} from '@extollo/lib'
import { Node } from "../models/Node.model";
import {Setting} from '../models/Setting.model'
import {IpRange} from '../models/IpRange.model'
import { Proxmox } from 'proxmox-api'
import {Images, LXCImage} from './Images.service'
import {unsafeESMImport} from '@extollo/lib/lib/util/unsafe'
import * as sshpk from 'sshpk'
import {shellCommand, SSHHost} from '../support/hosts'
@Singleton()
export class Provisioner {
@Inject()
protected readonly images!: Images
@Inject()
protected readonly container!: Container
@Inject()
protected readonly logging!: Logging
private cachedProxmoxApi?: Proxmox.Api
public async provisionMasterNode(): Promise<Node> {
// Look up the name of the master node from the settings
const pveMasterNode = await Setting.loadOneRequired('pveMasterNode')
// Look up the P5x storage pool from the settings
const pveStoragePool = await Setting.loadOneRequired('pveStoragePool')
// Look up the network bridge to which we will join the master node
const nodeNetworkBridge = await Setting.loadOneRequired('nodeNetworkBridge')
// Look up the primary IP pool range
const ipRange = await IpRange.getDefault()
// Look up the DNS search domain
const dnsDomain = await Setting.loadOneRequired('dnsDomain')
// Look up the root password for LXC containers
const rootPassword = await Setting.loadOneRequired('rootPassword')
// Get the Proxmox API
const proxmox = await this.getApi()
// Grab the master Proxmox node from the API
const masterNode = proxmox.nodes.$(pveMasterNode)
// Create a storage mount for the LXC template image
this.logging.info('Creating storage for container templates...')
const imageCacheStorage = await proxmox.storage.$post({
storage: 'p5x.image-cache',
type: 'dir',
content: ['vztmpl', 'iso'].join(','),
path: '/p5x-images',
})
const masterNodeStorage = masterNode.storage.$(imageCacheStorage.storage)
// Download the container template image
this.logging.info('Looking up LXC image for control node...')
const image = await this.images.getImages()
.then(x =>
x.where('os', '=', 'ubuntu')
.where('version', '=', 'jammy') // 22.04 LTS
.where('format', '=', 'cloud')
.where('arch', '=', 'amd64')
.sortByDesc('timestamp')
.first()
)
if ( !image ) {
throw new ErrorWithContext('Unable to find Debian Bookworm LXC template binary')
}
const imageUrl = this.images.getImageBinaryPath(image!)
// Upload the template image to the storage mount
const uploadTaskUPID = await masterNodeStorage['download-url'].$post({
content: 'vztmpl',
filename: 'p5x-base.tar.xz',
url: imageUrl.toRemote,
})
this.logging.info('Waiting for LXC image to upload to master PVE node...')
await this.waitForNodeTask(pveMasterNode, uploadTaskUPID)
// Get the next vmid to be used by the LXC container
const masterVMID = await proxmox.cluster.nextid.$get()
const masterIP = await ipRange.getNextAvailableOrFail()
// Create a new LXC container on that node for the control server
const sshPubKey = sshpk.parseKey(await Setting.loadOneRequired('sshPublicKey'))
.toString('ssh')
const masterTaskUPID = await masterNode.lxc.$post({
ostemplate: 'p5x.image-cache:vztmpl/p5x-base.tar.xz',
vmid: masterVMID,
cores: 1,
description: 'P5x Control Node',
hostname: 'p5x-control',
memory: 512, // in MB
net0:
objectToKeyValue({
name: 'eth0',
bridge: nodeNetworkBridge,
firewall: 1,
gw: ipRange.gatewayIp,
ip: `${masterIP}/${ipRange.subnet}`,
})
.map(x => `${x.key}=${x.value}`)
.join(','),
onboot: true,
password: rootPassword,
rootfs: `${pveStoragePool}:8`, // 8 GiB
searchdomain: dnsDomain,
'ssh-public-keys': sshPubKey,
start: true,
storage: pveStoragePool,
tags: 'p5x',
})
this.logging.info('Waiting for PVE master node to be created...')
await this.waitForNodeTask(pveMasterNode, masterTaskUPID)
// Save the node
const node = this.container.makeNew<Node>(Node)
node.pveId = masterVMID
node.hostname = 'p5x-control'
node.assignedIp = masterIP
node.assignedSubnet = ipRange.subnet
node.isPermanent = true
node.isMaster = true
await node.save()
// Wait for the host to come online (this will be a PCTHost passed through PVE)
this.logging.info('Waiting for master node to come online...')
const proxyHost = await node.getHost()
await proxyHost.waitForAlive()
// Configure SSH on the instance
this.logging.info('Configuring SSH access to master node...')
await proxyHost.run(shellCommand('apt update'))
await proxyHost.run(shellCommand('apt install -y openssh-server curl'))
await proxyHost.run(shellCommand('systemctl enable --now ssh'))
await new Promise<void>(res => setTimeout(() => res(), 5000))
// Try to re-connect to the instance directly
this.logging.info('Attempting to re-connect to master node via SSH...')
const sshHost = await node.getHost()
if ( !(sshHost instanceof SSHHost) ) {
throw new ErrorWithContext('Failed to establish direct connection to master node', {
node,
proxyHost,
sshHost,
})
}
// Install base environment needed to run this control server
await sshHost.run(shellCommand('curl -fsSL https://deb.nodesource.com/setup_20.x | bash -'))
await sshHost.run(shellCommand('apt install git nodejs -y'))
return node
}
protected async waitForNodeTask(nodeName: string, taskUPID: string): Promise<void> {
const proxmox = await this.getApi()
const proxmoxNode = proxmox.nodes.$(nodeName)
while ( true ) {
const status = await proxmoxNode.tasks.$(taskUPID).status.$get()
if ( status.status === 'running' ) {
await new Promise(res => setTimeout(res, 1000))
continue
}
if ( status.exitstatus !== 'OK' ) {
throw new ErrorWithContext('PVE task did not succeed!', {
nodeName,
taskUPID,
status,
})
}
break
}
}
public async getApi(): Promise<Proxmox.Api> {
// Have to force this to be a runtime ESM import since ts-node
// is re-writing it to a require(...), breaking the package.
const proxmoxApi = await unsafeESMImport('proxmox-api').then(x => x.proxmoxApi)
// If we have a cached version, just return that
if ( this.cachedProxmoxApi ) {
return this.cachedProxmoxApi
}
// Look up the master Proxmox node from the settings
const pveHost = await Setting.loadOneRequired('pveApiHost')
// Look up the Proxmox credentials from the settings
// const pveTokenId = await Setting.loadOneRequired('pveTokenId')
// const pveTokenSecret = await Setting.loadOneRequired('pveTokenSecret')
const pveRootPassword = await Setting.loadOneRequired('pveRootPassword')
// Initialize the Proxmox API
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
this.cachedProxmoxApi = proxmoxApi({
host: pveHost,
user: 'root@pam',
password: pveRootPassword,
// tokenID: pveTokenId,
// tokenSecret: pveTokenSecret,
}) as Proxmox.Api
return this.cachedProxmoxApi
}
}

View File

@ -0,0 +1,36 @@
export class ExecutionResult {
protected stdout: string[] = []
protected stderr: string[] = []
protected mixed: string[] = []
public exitCode?: number
out(data: any) {
this.stdout = this.stdout.concat(`${data}`.split('\n'))
this.mixed = this.mixed.concat(`${data}`.split('\n'))
}
error(data: any) {
this.stderr = this.stderr.concat(`${data}`.split('\n'))
this.mixed = this.mixed.concat(`${data}`.split('\n'))
}
exit(code: number) {
this.exitCode = code
}
public wasSuccessful(): boolean {
return this.exitCode === 0
}
public get combinedOutput(): string[] {
return this.mixed.filter(Boolean)
}
public get standardOutput(): string[] {
return this.stdout.filter(Boolean)
}
public get errorOutput(): string[] {
return this.stderr.filter(Boolean)
}
}

View File

@ -0,0 +1,203 @@
import {
Awaitable,
AwareOfContainerLifecycle, ErrorWithContext,
Filesystem,
infer,
objectToKeyValue,
UniversalPath,
uuid4,
} from '@extollo/lib'
import {CommandPalette, CommandPaletteCommands, OwnershipSpec, shellCommand, ShellCommand} from './types'
import {ExecutionResult} from './ExecutionResult'
import {CommandError} from './errors'
import {SystemMetrics} from './SystemMetrics'
export class StandardCommandPalette implements CommandPalette {
changeDirectory = shellCommand(`cd "%%PATH%%"`)
cpuPercentage = shellCommand(`grep 'cpu ' /proc/stat | awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage}'`)
fileDirectoryDelete = shellCommand(`rm -rf "%%RESOURCE%%"`)
fileDirectoryOwnershipFetch = shellCommand(`stat -c "%U:%G" "%%PATH%%"`)
fileDirectoryOwnershipSetFlat = shellCommand(`chown %%OWNERS%% "%%PATH%%"`)
fileDirectoryOwnershipSetRecursive = shellCommand(`chown -R %%OWNERS%% "%%PATH%%"`)
fileDirectoryPermissionFetch = shellCommand(`stat -c %a "%%PATH%%"`)
fileDirectoryPermissionSetFlat = shellCommand(`chmod %%LEVEL%% "%%PATH%%"`)
fileDirectoryPermissionSetRecursive = shellCommand(`chmod -R %%LEVEL%% "%%PATH%%"`)
listMountPoints = shellCommand(`df -hl | grep '/' | awk '{print $6}`)
mountPointPercentage = shellCommand(`df -hl | grep -w '%%MOUNTPOINT%%$' | awk '{print $5}'`)
ramPercentage = shellCommand(`free | grep Mem | awk '{print $3/$2 * 100.0}'`)
reboot = shellCommand(`reboot`)
resolvePath = shellCommand(`readlink -f "%%PATH%%"`)
tempFile = shellCommand(`mktemp`)
tempPath = shellCommand(`mktemp -d`)
echo = shellCommand(`echo "%%OUTPUT%%"`)
public format(command: keyof CommandPaletteCommands, substitutions: Record<string, any>): ShellCommand {
let formatted = `${this[command]}`
objectToKeyValue(substitutions)
.map(item =>
formatted = formatted.replaceAll(`%%${item.key.toUpperCase()}%%`, `${item.value}`))
return shellCommand(formatted)
}
}
export abstract class Host implements AwareOfContainerLifecycle {
awareOfContainerLifecycle: true = true
public abstract getFilesystem(): Awaitable<Filesystem>
public abstract execute(command: ShellCommand): Awaitable<ExecutionResult>
close(): Awaitable<void> {}
protected getCommandPalette(): CommandPalette {
return new StandardCommandPalette()
}
onContainerDestroy() {
this.close()
}
onContainerRelease() {
this.close()
}
/** Runs the provided command and throws if it failed. */
public async run(command: ShellCommand): Promise<ExecutionResult> {
const result = await this.execute(command)
if ( !result.wasSuccessful() ) {
throw CommandError.make(command, result)
}
return result
}
/** Runs the provided command, throws if it failed, and returns the `infer(...)` result of the first line otherwise. */
public async runLineResult(command: ShellCommand): Promise<ReturnType<typeof infer>> {
const result = await this.execute(command)
if ( !result.wasSuccessful() || result.standardOutput.length < 1 ) {
throw CommandError.make(command, result)
}
return infer(result.standardOutput[0].trim())
}
/** Try to ping the host and verify that it is online & accessible. */
public async isAlive(): Promise<boolean> {
const output = uuid4()
const cmd = this.getCommandPalette().format('echo', { output })
try {
const result = await this.execute(cmd)
return result.wasSuccessful() && result.combinedOutput.length > 0 && result.combinedOutput[0] === output
} catch (e) {
return false
}
}
public async waitForAlive(maxTries: number = 30): Promise<void> {
let tries = maxTries
while ( tries > 0 ) {
if ( await this.isAlive() ) {
return
}
tries -= 1
await new Promise<void>(res => setTimeout(() => res(), 2000))
}
throw new ErrorWithContext('Host did not come online before maxTries exceeded', {
host: this,
maxTries,
})
}
/** Get the path to a temporary directory on the host. */
public async getTempDirectory(): Promise<UniversalPath> {
const cmd = this.getCommandPalette().format('tempPath', {})
const result = await this.runLineResult(cmd)
return new UniversalPath(`${result}`, await this.getFilesystem())
}
/** Get the path to a temporary file on the host. */
public async getTempFile(): Promise<UniversalPath> {
const cmd = this.getCommandPalette().format('tempFile', {})
const result = await this.runLineResult(cmd)
return new UniversalPath(`${result}`, await this.getFilesystem())
}
/** Get a list of filesystem mount points on the host. */
public async getMountPoints(): Promise<string[]> {
const result = await this.run(this.getCommandPalette().format('listMountPoints', {}))
return result.standardOutput
}
/** Load standard metrics about the host. */
public async metrics(): Promise<SystemMetrics> {
const metric = new SystemMetrics()
const palette = this.getCommandPalette()
const cpuCmd = palette.format('cpuPercentage', {})
const cpuResult = await this.runLineResult(cpuCmd)
if ( typeof cpuResult === 'number' ) {
metric.cpuPercent = cpuResult
}
const ramCmd = palette.format('ramPercentage', {})
const ramResult = await this.runLineResult(ramCmd)
if ( typeof ramResult === 'number' ) {
metric.ramPercent = ramResult
}
const mountPoints = await this.getMountPoints()
await Promise.all(mountPoints.map(async mountPoint => {
const cmd = palette.format('mountPointPercentage', { mountPoint })
const result = await this.run(cmd)
const utilization = Number(result.standardOutput[0].replace('%', '')) / 100
metric.setMount(mountPoint, utilization)
}))
return metric
}
/** Get the user/group ownership of the given resource. */
public async getOwnership(path: UniversalPath): Promise<OwnershipSpec> {
const cmd = this.getCommandPalette().format('fileDirectoryOwnershipFetch', { path: path.toLocal })
const result = `${await this.runLineResult(cmd)}`
const parts = result.split(':')
return {user: parts[0], group: parts[1]}
}
/** Set the user/group ownership of the given resource. */
public setOwnership(path: UniversalPath, user: string, group: string, recursive: boolean = false): Awaitable<ExecutionResult> {
const cmd = this.getCommandPalette().format(
recursive ? 'fileDirectoryOwnershipSetRecursive' : 'fileDirectoryOwnershipSetFlat',
{
path: path.toLocal,
owners: `${user}:${group}`,
}
)
return this.run(cmd)
}
/** Retrieve the filesystem permissions for the given resource. */
public async getPermissions(path: UniversalPath): Promise<string> {
const cmd = this.getCommandPalette().format('fileDirectoryPermissionFetch', { path: path.toLocal })
return `${await this.runLineResult(cmd)}`
}
/** Set the filesystem permissions for the given resource. */
public setPermissions(path: UniversalPath, level: string, recursive: boolean = false): Awaitable<ExecutionResult> {
const cmd = this.getCommandPalette().format(
recursive ? 'fileDirectoryPermissionSetRecursive' : 'fileDirectoryPermissionSetFlat',
{
path: path.toLocal,
level,
}
)
return this.run(cmd)
}
/** Reboot the host. */
public reboot(): Awaitable<ExecutionResult> {
return this.run(this.getCommandPalette().format('reboot', {}))
}
}

View File

@ -0,0 +1,25 @@
import {Host} from './Host'
import {ShellCommand} from './types'
import {Application, Awaitable, Filesystem, LocalFilesystem} from '@extollo/lib'
import {ExecutionResult} from './ExecutionResult'
import * as childProcess from 'child_process'
export class LocalHost extends Host {
async execute(command: ShellCommand): Promise<ExecutionResult> {
const result = new ExecutionResult()
return new Promise(res => {
childProcess.exec(command, (error, stdout, stderr) => {
result.exit(error?.code || 0)
result.out(stdout)
result.error(stderr)
res(result)
})
})
}
getFilesystem(): Awaitable<Filesystem> {
return Application.getContainer().makeNew<LocalFilesystem>(LocalFilesystem, {
baseDir: '/'
})
}
}

View File

@ -0,0 +1,24 @@
import {SSHHost} from './SSHHost'
import * as ssh2 from 'ssh2'
import {shellCommand, ShellCommand} from './types'
import {ExecutionResult} from './ExecutionResult'
import {ErrorWithContext, Filesystem} from '@extollo/lib'
export class PCTHost extends SSHHost {
constructor(
config: ssh2.ConnectConfig,
protected readonly vmid: number,
) {
super(config)
}
async execute(command: ShellCommand): Promise<ExecutionResult> {
return super.execute(shellCommand('pct exec', `${this.vmid}`, '--', command))
}
getFilesystem(): Promise<Filesystem>{
throw new ErrorWithContext('Accessing Filesystem of PCTHost is not supported.', {
vmid: this.vmid,
})
}
}

View File

@ -0,0 +1,77 @@
import {Host} from './Host'
import {Application, Awaitable, Filesystem, SSHFilesystem} from '@extollo/lib'
import {ExecutionResult} from './ExecutionResult'
import {ShellCommand} from './types'
import * as ssh2 from 'ssh2'
export class SSHHost extends Host {
private sshClient?: ssh2.Client
private filesystem?: Filesystem
constructor(
protected readonly config: ssh2.ConnectConfig,
) {
super()
}
async close() {
this.sshClient?.end()
this.sshClient?.destroy()
this.sshClient = undefined
this.filesystem?.close()
this.filesystem = undefined
}
async execute(command: ShellCommand): Promise<ExecutionResult>{
const client = await this.getSSH()
const result = new ExecutionResult()
return new Promise<ExecutionResult>((res, rej) => {
client.exec(command, (err, stream) => {
if ( err ) {
return rej(err)
}
stream
.on('close', (code: number) => {
result.exit(code)
res(result)
})
.on('data', (data: any) => {
result.out(data)
})
.stderr.on('data', (data: any) => {
result.error(data)
})
})
})
}
getFilesystem(): Awaitable<Filesystem> {
if ( !this.filesystem ) {
this.filesystem = Application.getContainer().makeNew<SSHFilesystem>(SSHFilesystem, {
ssh: this.config,
baseDir: '/',
})
}
return this.filesystem
}
protected async getSSH(): Promise<ssh2.Client> {
if ( this.sshClient ) {
return this.sshClient
}
return new Promise((res, rej) => {
const client = new ssh2.Client()
client.on('ready', () => {
this.sshClient = client
res(client)
}).connect(this.config)
client.on('error', rej)
})
}
}

View File

@ -0,0 +1,15 @@
import {Maybe} from '@extollo/lib'
export class SystemMetrics {
public cpuPercent: number = 0
public ramPercent: number = 0
protected mountPercents: Record<string, number> = {}
public setMount(point: string, percent: number) {
this.mountPercents[point] = percent
}
public mountPercent(point: string): Maybe<number> {
return this.mountPercents[point]
}
}

View File

@ -0,0 +1,51 @@
import {ErrorWithContext} from '@extollo/lib'
import {ShellCommand} from './types'
import {ExecutionResult} from './ExecutionResult'
export class CommandError extends ErrorWithContext {
public static make(command: ShellCommand, result: ExecutionResult): CommandError {
const combinedOutput = result.combinedOutput.join(' ').toLowerCase()
const accessDeniedPhrases = [
'not permitted',
'access denied',
'insufficient permission',
'permission denied'
]
for ( const phrase of accessDeniedPhrases ) {
if ( combinedOutput.includes(phrase) ) return new PermissionDeniedError(command, result)
}
const notFoundPhrases = [
'command not found',
]
for ( const phrase of notFoundPhrases ) {
if ( combinedOutput.includes(phrase) ) return new CommandNotFoundError(command, result)
}
const resourceNotFoundPhrases = [
'no such file',
'no such directory',
'no such file or directory',
'no such directory or file',
]
for ( const phrase of resourceNotFoundPhrases ) {
if ( combinedOutput.includes(phrase) ) return new FilesystemResourceNotFoundError(command, result)
}
return new CommandError(command, result)
}
constructor(command: ShellCommand, result: ExecutionResult) {
super(`Unable to execute command: ${command}`, { command, result })
}
}
export class PermissionDeniedError extends CommandError {}
export class CommandNotFoundError extends CommandError {}
export class FilesystemResourceNotFoundError extends CommandError {}

View File

@ -0,0 +1,7 @@
export * from './types'
export * from './errors'
export * from './ExecutionResult'
export * from './SystemMetrics'
export * from './Host'
export * from './LocalHost'
export * from './SSHHost'

View File

@ -0,0 +1,34 @@
import {TypeTag} from '@extollo/lib'
export type ShellCommand = TypeTag<'p5x.shell-command'> & string
export const shellCommand = (...parts: string[]): ShellCommand => parts.join(' ') as ShellCommand
export interface CommandPaletteCommands {
tempPath: ShellCommand
tempFile: ShellCommand
cpuPercentage: ShellCommand
ramPercentage: ShellCommand
mountPointPercentage: ShellCommand
listMountPoints: ShellCommand
fileDirectoryDelete: ShellCommand
resolvePath: ShellCommand
reboot: ShellCommand
changeDirectory: ShellCommand
fileDirectoryPermissionFetch: ShellCommand
fileDirectoryPermissionSetFlat: ShellCommand
fileDirectoryPermissionSetRecursive: ShellCommand
fileDirectoryOwnershipFetch: ShellCommand
fileDirectoryOwnershipSetFlat: ShellCommand
fileDirectoryOwnershipSetRecursive: ShellCommand
echo: ShellCommand
}
export type CommandPalette = CommandPaletteCommands & {
format(command: keyof CommandPaletteCommands, substitutions: Record<string, any>): ShellCommand
}
export interface OwnershipSpec {
user: string
group: string
}

31
src/app/types.ts Normal file
View File

@ -0,0 +1,31 @@
import {TypeTag, JSONState, isJSON} from '@extollo/lib'
import * as IPCIDR from 'ip-cidr'
export type IpAddress = TypeTag<'p5x.ip.address'> & string
export const isIpAddress = (what: unknown): what is IpAddress =>
typeof what === 'string' && IPCIDR.isValidAddress(what)
export type Subnet = TypeTag<'p5x.ip.subnet'> & number
export const isSubnet = (what: unknown): what is Subnet =>
typeof what === 'number' && IPCIDR.isValidCIDR(`1.1.1.1/${what}`)
export type JSONString = TypeTag<'p5x.json'> & string
export const isJSONString = (what: unknown): what is JSONString =>
typeof what === 'string' && isJSON(what)
export type Settings = Record<string, JSONState> & {
pveMasterNode: string
pveApiHost: string
pveStoragePool: string
// pveTokenId: string
// pveTokenSecret: string
pveRootPassword: string
lxcTemplateFileName: string
rootPassword: string
sshPublicKey: string
sshPrivateKey: string
dnsDomain: string
nodeNetworkBridge: string
nodeCpus: number
nodeRamMib: number
}

View File

@ -12,4 +12,5 @@ globalRegistry.run(async () => {
*/
const app = cli()
await app.run()
await app.destroy()
})

View File

@ -9,5 +9,7 @@ globalRegistry.run(async () => {
* ties your entire application together. The app container manages services
* and lifecycle.
*/
await app().run()
const theApp = app()
await theApp.run()
await theApp.destroy()
})