import { Singleton, Inject, ErrorWithContext, objectToKeyValue, fetch, Container, Logging, universalPath, } from '@extollo/lib' import { Node } from "../models/Node.model"; import {Setting} from '../models/Setting.model' import {IpRange} from '../models/IpRange.model' import { Proxmox } from 'proxmox-api' import {Images, LXCImage} from './Images.service' import {unsafeESMImport} from '@extollo/lib/lib/util/unsafe' import * as sshpk from 'sshpk' import {shellCommand, SSHHost} from '../support/hosts' @Singleton() export class Provisioner { @Inject() protected readonly images!: Images @Inject() protected readonly container!: Container @Inject() protected readonly logging!: Logging private cachedProxmoxApi?: Proxmox.Api public async provisionMasterNode(): Promise { // Look up the name of the master node from the settings const pveMasterNode = await Setting.loadOneRequired('pveMasterNode') // Look up the P5x storage pool from the settings const pveStoragePool = await Setting.loadOneRequired('pveStoragePool') // Look up the network bridge to which we will join the master node const nodeNetworkBridge = await Setting.loadOneRequired('nodeNetworkBridge') // Look up the primary IP pool range const ipRange = await IpRange.getDefault() // Look up the DNS search domain const dnsDomain = await Setting.loadOneRequired('dnsDomain') // Look up the root password for LXC containers const rootPassword = await Setting.loadOneRequired('rootPassword') // Get the Proxmox API const proxmox = await this.getApi() // Grab the master Proxmox node from the API const masterNode = proxmox.nodes.$(pveMasterNode) // Create a storage mount for the LXC template image this.logging.info('Creating storage for container templates...') const imageCacheStorage = await proxmox.storage.$post({ storage: 'p5x.image-cache', type: 'dir', content: ['vztmpl', 'iso'].join(','), path: '/p5x-images', }) const masterNodeStorage = masterNode.storage.$(imageCacheStorage.storage) // Download the container template image this.logging.info('Looking up LXC image for control node...') const image = await this.images.getImages() .then(x => x.where('os', '=', 'ubuntu') .where('version', '=', 'jammy') // 22.04 LTS .where('format', '=', 'cloud') .where('arch', '=', 'amd64') .sortByDesc('timestamp') .first() ) if ( !image ) { throw new ErrorWithContext('Unable to find Debian Bookworm LXC template binary') } const imageUrl = this.images.getImageBinaryPath(image!) // Upload the template image to the storage mount const uploadTaskUPID = await masterNodeStorage['download-url'].$post({ content: 'vztmpl', filename: 'p5x-base.tar.xz', url: imageUrl.toRemote, }) this.logging.info('Waiting for LXC image to upload to master PVE node...') await this.waitForNodeTask(pveMasterNode, uploadTaskUPID) // Get the next vmid to be used by the LXC container const masterVMID = await proxmox.cluster.nextid.$get() const masterIP = await ipRange.getNextAvailableOrFail() // Create a new LXC container on that node for the control server const sshPubKey = sshpk.parseKey(await Setting.loadOneRequired('sshPublicKey')) .toString('ssh') const masterTaskUPID = await masterNode.lxc.$post({ ostemplate: 'p5x.image-cache:vztmpl/p5x-base.tar.xz', vmid: masterVMID, cores: 1, description: 'P5x Control Node', hostname: 'p5x-control', memory: 512, // in MB net0: objectToKeyValue({ name: 'eth0', bridge: nodeNetworkBridge, firewall: 1, gw: ipRange.gatewayIp, ip: `${masterIP}/${ipRange.subnet}`, }) .map(x => `${x.key}=${x.value}`) .join(','), onboot: true, password: rootPassword, rootfs: `${pveStoragePool}:8`, // 8 GiB searchdomain: dnsDomain, 'ssh-public-keys': sshPubKey, start: true, storage: pveStoragePool, tags: 'p5x', }) this.logging.info('Waiting for PVE master node to be created...') await this.waitForNodeTask(pveMasterNode, masterTaskUPID) // Save the node const node = this.container.makeNew(Node) node.pveId = masterVMID node.hostname = 'p5x-control' node.assignedIp = masterIP node.assignedSubnet = ipRange.subnet node.isPermanent = true node.isMaster = true await node.save() // Wait for the host to come online (this will be a PCTHost passed through PVE) this.logging.info('Waiting for master node to come online...') const proxyHost = await node.getHost() await proxyHost.waitForAlive() // Configure SSH on the instance this.logging.info('Configuring SSH access to master node...') await proxyHost.run(shellCommand('apt update')) await proxyHost.run(shellCommand('apt install -y openssh-server curl')) await proxyHost.run(shellCommand('systemctl enable --now ssh')) await new Promise(res => setTimeout(() => res(), 5000)) // Try to re-connect to the instance directly this.logging.info('Attempting to re-connect to master node via SSH...') const sshHost = await node.getHost() if ( !(sshHost instanceof SSHHost) ) { throw new ErrorWithContext('Failed to establish direct connection to master node', { node, proxyHost, sshHost, }) } // Install base environment needed to run this control server await sshHost.run(shellCommand('curl -fsSL https://deb.nodesource.com/setup_20.x | bash -')) await sshHost.run(shellCommand('apt install git nodejs -y')) return node } protected async waitForNodeTask(nodeName: string, taskUPID: string): Promise { const proxmox = await this.getApi() const proxmoxNode = proxmox.nodes.$(nodeName) while ( true ) { const status = await proxmoxNode.tasks.$(taskUPID).status.$get() if ( status.status === 'running' ) { await new Promise(res => setTimeout(res, 1000)) continue } if ( status.exitstatus !== 'OK' ) { throw new ErrorWithContext('PVE task did not succeed!', { nodeName, taskUPID, status, }) } break } } public async getApi(): Promise { // Have to force this to be a runtime ESM import since ts-node // is re-writing it to a require(...), breaking the package. const proxmoxApi = await unsafeESMImport('proxmox-api').then(x => x.proxmoxApi) // If we have a cached version, just return that if ( this.cachedProxmoxApi ) { return this.cachedProxmoxApi } // Look up the master Proxmox node from the settings const pveHost = await Setting.loadOneRequired('pveApiHost') // Look up the Proxmox credentials from the settings // const pveTokenId = await Setting.loadOneRequired('pveTokenId') // const pveTokenSecret = await Setting.loadOneRequired('pveTokenSecret') const pveRootPassword = await Setting.loadOneRequired('pveRootPassword') // Initialize the Proxmox API process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; this.cachedProxmoxApi = proxmoxApi({ host: pveHost, user: 'root@pam', password: pveRootPassword, // tokenID: pveTokenId, // tokenSecret: pveTokenSecret, }) as Proxmox.Api return this.cachedProxmoxApi } }