control-server/src/app/services/Provisioner.service.ts

import {
    Singleton,
    Inject,
    ErrorWithContext,
    objectToKeyValue,
    fetch,
    Container,
    Logging,
    universalPath,
} from '@extollo/lib'
import { Node } from "../models/Node.model";
import {Setting} from '../models/Setting.model'
import {IpRange} from '../models/IpRange.model'
import { Proxmox } from 'proxmox-api'
import {Images, LXCImage} from './Images.service'
import {unsafeESMImport} from '@extollo/lib/lib/util/unsafe'
import * as sshpk from 'sshpk'
import {shellCommand, SSHHost} from '../support/hosts'

@Singleton()
export class Provisioner {
    @Inject()
    protected readonly images!: Images

    @Inject()
    protected readonly container!: Container

    @Inject()
    protected readonly logging!: Logging

    private cachedProxmoxApi?: Proxmox.Api

    public async provisionMasterNode(): Promise<Node> {
        // Look up the name of the master node from the settings
        const pveMasterNode = await Setting.loadOneRequired('pveMasterNode')

        // Look up the P5x storage pool from the settings
        const pveStoragePool = await Setting.loadOneRequired('pveStoragePool')

        // Look up the network bridge to which we will join the master node
        const nodeNetworkBridge = await Setting.loadOneRequired('nodeNetworkBridge')

        // Look up the primary IP pool range
        const ipRange = await IpRange.getDefault()

        // Look up the DNS search domain
        const dnsDomain = await Setting.loadOneRequired('dnsDomain')

        // Look up the root password for LXC containers
        const rootPassword = await Setting.loadOneRequired('rootPassword')

        // Get the Proxmox API
        const proxmox = await this.getApi()

        // Grab the master Proxmox node from the API
        const masterNode = proxmox.nodes.$(pveMasterNode)

        // Create a storage mount for the LXC template image
        this.logging.info('Creating storage for container templates...')
        const imageCacheStorage = await proxmox.storage.$post({
            storage: 'p5x.image-cache',
            type: 'dir',
            content: ['vztmpl', 'iso'].join(','),
            path: '/p5x-images',
        })
        const masterNodeStorage = masterNode.storage.$(imageCacheStorage.storage)

        // Download the container template image
        this.logging.info('Looking up LXC image for control node...')
        const image = await this.images.getImages()
            .then(x =>
                x.where('os', '=', 'ubuntu')
                    .where('version', '=', 'jammy')  // 22.04 LTS
                    .where('format', '=', 'cloud')
                    .where('arch', '=', 'amd64')
                    .sortByDesc('timestamp')
                    .first()
            )

        if ( !image ) {
            throw new ErrorWithContext('Unable to find Debian Bookworm LXC template binary')
        }

        const imageUrl = this.images.getImageBinaryPath(image!)

        // Upload the template image to the storage mount
        const uploadTaskUPID = await masterNodeStorage['download-url'].$post({
            content: 'vztmpl',
            filename: 'p5x-base.tar.xz',
            url: imageUrl.toRemote,
        })

        this.logging.info('Waiting for LXC image to upload to master PVE node...')
        await this.waitForNodeTask(pveMasterNode, uploadTaskUPID)

        // Get the next vmid to be used by the LXC container
        const masterVMID = await proxmox.cluster.nextid.$get()
        const masterIP = await ipRange.getNextAvailableOrFail()

        // Create a new LXC container on that node for the control server
        const sshPubKey = sshpk.parseKey(await Setting.loadOneRequired('sshPublicKey'))
            .toString('ssh')

        const masterTaskUPID = await masterNode.lxc.$post({
            ostemplate: 'p5x.image-cache:vztmpl/p5x-base.tar.xz',
            vmid: masterVMID,
            cores: 1,
            description: 'P5x Control Node',
            hostname: 'p5x-control',
            memory: 512,  // in MB
            net0:
                objectToKeyValue({
                    name: 'eth0',
                    bridge: nodeNetworkBridge,
                    firewall: 1,
                    gw: ipRange.gatewayIp,
                    ip: `${masterIP}/${ipRange.subnet}`,
                })
                .map(x => `${x.key}=${x.value}`)
                .join(','),
            onboot: true,
            password: rootPassword,
            rootfs: `${pveStoragePool}:8`,  // 8 GiB
            searchdomain: dnsDomain,
            'ssh-public-keys': sshPubKey,
            start: true,
            storage: pveStoragePool,
            tags: 'p5x',
        })

        this.logging.info('Waiting for PVE master node to be created...')
        await this.waitForNodeTask(pveMasterNode, masterTaskUPID)

        // Save the node
        const node = this.container.makeNew<Node>(Node)
        node.pveId = masterVMID
        node.hostname = 'p5x-control'
        node.assignedIp = masterIP
        node.assignedSubnet = ipRange.subnet
        node.isPermanent = true
        node.isMaster = true
        await node.save()

        // Wait for the host to come online (this will be a PCTHost passed through PVE)
        this.logging.info('Waiting for master node to come online...')
        const proxyHost = await node.getHost()
        await proxyHost.waitForAlive()

        // Configure SSH on the instance
        this.logging.info('Configuring SSH access to master node...')
        await proxyHost.run(shellCommand('apt update'))
        await proxyHost.run(shellCommand('apt install -y openssh-server curl'))
        await proxyHost.run(shellCommand('systemctl enable --now ssh'))
        await new Promise<void>(res => setTimeout(() => res(), 5000))

        // Try to re-connect to the instance directly
        this.logging.info('Attempting to re-connect to master node via SSH...')
        const sshHost = await node.getHost()
        if ( !(sshHost instanceof SSHHost) ) {
            throw new ErrorWithContext('Failed to establish direct connection to master node', {
                node,
                proxyHost,
                sshHost,
            })
        }

        // Install base environment needed to run this control server
        await sshHost.run(shellCommand('curl -fsSL https://deb.nodesource.com/setup_20.x | bash -'))
        await sshHost.run(shellCommand('apt install git nodejs -y'))

        return node
    }

    protected async waitForNodeTask(nodeName: string, taskUPID: string): Promise<void> {
        const proxmox = await this.getApi()
        const proxmoxNode = proxmox.nodes.$(nodeName)

        while ( true ) {
            const status = await proxmoxNode.tasks.$(taskUPID).status.$get()
            if ( status.status === 'running' ) {
                await new Promise(res => setTimeout(res, 1000))
                continue
            }

            if ( status.exitstatus !== 'OK' ) {
                throw new ErrorWithContext('PVE task did not succeed!', {
                    nodeName,
                    taskUPID,
                    status,
                })
            }

            break
        }
    }

    public async getApi(): Promise<Proxmox.Api> {
        // Have to force this to be a runtime ESM import since ts-node
        // is re-writing it to a require(...), breaking the package.
        const proxmoxApi = await unsafeESMImport('proxmox-api').then(x => x.proxmoxApi)

        // If we have a cached version, just return that
        if ( this.cachedProxmoxApi ) {
            return this.cachedProxmoxApi
        }

        // Look up the master Proxmox node from the settings
        const pveHost = await Setting.loadOneRequired('pveApiHost')

        // Look up the Proxmox credentials from the settings
        // const pveTokenId = await Setting.loadOneRequired('pveTokenId')
        // const pveTokenSecret = await Setting.loadOneRequired('pveTokenSecret')
        const pveRootPassword = await Setting.loadOneRequired('pveRootPassword')

        // Initialize the Proxmox API
        process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
        this.cachedProxmoxApi = proxmoxApi({
            host: pveHost,
            user: 'root@pam',
            password: pveRootPassword,
            // tokenID: pveTokenId,
            // tokenSecret: pveTokenSecret,
        }) as Proxmox.Api

        return this.cachedProxmoxApi
    }

}