[WIP] Early provisioning implementation

src/app/cli/directive/StandupDirective.ts

@@ -62,7 +62,7 @@ export class StandupDirective extends Directive {
         ipRange.endIp = this.getIpOptionOrFail('ip-pool-end')
         ipRange.subnet = this.getSubnetOptionOrFail('ip-pool-subnet')
         ipRange.gatewayIp = this.getIpOptionOrFail('ip-pool-gateway')
-        ipRange.save()
+        await ipRange.save()
 
         // Provision the master node:
         const masterNode = await this.provisioner.provisionMasterNode()

src/app/cobalt.ts

@@ -70,7 +70,7 @@ export type RemoteSelectDefinition = {
 }
 
 export type FieldDefinition = FieldBase
-    | FieldBase & { type: FieldType.select, options: SelectOptions }
+    | FieldBase & { type: FieldType.select, multiple?: boolean, options: SelectOptions }
     | FieldBase & { type: FieldType.select } & RemoteSelectDefinition
 
 export interface DataSourceController {

src/app/configs/cobalt.config.ts

@@ -42,6 +42,7 @@ export default {
                     display: 'Physical Hosts',
                     type: FieldType.select,
                     required: true,
+                    multiple: true,
                     displayFrom: 'pveDisplay',
                     valueFrom: 'pveHost',
                     hideOn: {

src/app/http/controllers/Dash.controller.ts

@@ -2,21 +2,72 @@ import {
     Controller,
     Injectable,
     Inject,
-    Logging, ResponseObject, view,
+    Logging, ResponseObject, view, redirect,
 } from '@extollo/lib'
+import {ControlPlane} from '../../support/k8s/ControlPlane'
+import {Node} from '../../models/Node.model'
+import {Provisioner} from '../../services/Provisioner.service'
+import {HostGroup} from '../../models/HostGroup.model'
 
 @Injectable()
 export class Dash extends Controller {
     @Inject()
     protected readonly logging!: Logging
 
+    @Inject()
+    protected readonly provisioner!: Provisioner
+
     public main(): ResponseObject {
         return view('dash:template')
     }
 
-    public provision(): ResponseObject {
+    public async provision(): Promise<ResponseObject> {
+        const cd = await this.provisioner.provisionNode((await HostGroup.findByKey(2))!)
+        const cp = new ControlPlane(await Node.getMaster().then(m => m.getHost()))
+        const cv = await cp.initializeNode(cd)
+        console.log({cv})
         return view('dash:provision', {
 
         })
     }
+
+    public async controlPlane(): Promise<ResponseObject> {
+        const master = await Node.getMaster()
+        const plane = new ControlPlane(await master.getHost())
+
+        const hasKubelet = await plane.hasKubelet()
+        const extras: any = {}
+        if ( hasKubelet ) {
+            extras.version = await plane.getVersion()
+        }
+
+        return view('dash:control-plane', {
+            hasKubelet,
+            ...extras,
+        })
+    }
+
+    public async controlPlaneInitialize(): Promise<ResponseObject> {
+        const master = await Node.getMaster()
+        const plane = new ControlPlane(await master.getHost())
+
+        if ( await plane.hasKubelet() ) {
+            return redirect('/dash/control-plane')
+        }
+
+        await plane.initializeMaster()
+        return redirect('/dash/control-plane')
+    }
+
+    public async controlPlaneUpgrade(): Promise<ResponseObject> {
+        const master = await Node.getMaster()
+        const plane = new ControlPlane(await master.getHost())
+
+        if ( await plane.hasKubelet() ) {
+            await plane.initializeMaster()
+            return redirect('/dash/control-plane')
+        }
+
+        return redirect('/dash/control-plane')
+    }
 }

src/app/http/controllers/cobalt/ResourceAPI.controller.ts

@@ -10,7 +10,7 @@ import {
     HTTPError,
     HTTPStatus,
     Inject,
-    Injectable, Iterable,
+    Injectable, Iterable, JSONState,
     Maybe,
     QueryRow,
 } from '@extollo/lib'
@@ -273,7 +273,7 @@ export class ResourceAPI extends Controller {
                 throw new HTTPError(HTTPStatus.BAD_REQUEST, `Missing required field: ${key}`)
             }
 
-            if ( isNaN(cast) ) {
+            if ( value && value !== 0 && isNaN(cast) ) {
                 throw new HTTPError(HTTPStatus.BAD_REQUEST, `Field must be a number: ${key}`)
             }
             return cast
@@ -283,7 +283,7 @@ export class ResourceAPI extends Controller {
                 throw new HTTPError(HTTPStatus.BAD_REQUEST, `Missing required field: ${key}`)
             }
 
-            if ( isNaN(cast) ) {
+            if ( value && value !== 0 && isNaN(cast) ) {
                 throw new HTTPError(HTTPStatus.BAD_REQUEST, `Field must be a number: ${key}`)
             }
             return cast
@@ -308,8 +308,10 @@ export class ResourceAPI extends Controller {
             return cast
         } else if ( type === FieldType.select && hasOwnProperty(fieldDef, 'options') ) {
             const options = collect(fieldDef.options as SelectOptions)
-            if ( options.pluck('value').includes(value) ) {
+            const selectedValues = Array.isArray(value) ? value : [value]
-                return value
+            const validOptions = options.pluck('value').intersect(selectedValues)
+            if ( validOptions.isNotEmpty() ) {
+                return fieldDef.multiple ? validOptions.all() : validOptions.get(0)
             }
 
             if ( required ) {

src/app/http/controllers/resource/HostGroups.controller.ts

@@ -27,7 +27,7 @@ export class HostGroups extends Controller implements DataSourceController {
     insert(row: QueryRow): Awaitable<QueryRow> {
         const hosts = row.hosts
         if ( !Array.isArray(hosts) || !hosts.every(i => typeof i === 'string') ) {
-            throw new Error('Invalid hosts: must be number[]')
+            throw new Error('Invalid hosts: must be string[]')
         }
 
         return AsyncPipe.wrap(this.request.makeNew<HostGroup>(HostGroup))
@@ -59,7 +59,7 @@ export class HostGroups extends Controller implements DataSourceController {
     }
 
     async delete(id: any): Promise<void> {
-        HostGroup.query<HostGroup>()
+        await HostGroup.query<HostGroup>()
             .whereKey(id)
             .delete()
     }
@@ -69,10 +69,12 @@ export class HostGroups extends Controller implements DataSourceController {
             .tap(p => p.getApi())
             .tap(api => api.nodes.$get())
             .tap(nodes =>
-                nodes.map(node => ({
+                nodes
-                    pveHost: node.id || node.node,
+                    .map(node => ({
-                    pveDisplay: `${node.node} (CPUs: ${node.maxcpu}, RAM: ${Math.floor((node.maxmem || 0) / 1000000000)})`
+                        pveHost: node.id || node.node,
-                }))
+                        pveDisplay: `${node.node} (CPUs: ${node.maxcpu}, RAM: ${Math.floor((node.maxmem || 0) / 1000000000)})`
+                    }))
+                    .sort((a, b) => a.pveDisplay.localeCompare(b.pveDisplay))
             )
             .resolve()
     }

src/app/http/routes/dash.routes.ts

@@ -33,6 +33,15 @@ Route
         Route.get('/provision')
             .calls<Dash>(Dash, dash => dash.provision)
 
+        Route.get('/control-plane')
+            .calls<Dash>(Dash, dash => dash.controlPlane)
+
+        Route.get('/control-plane/initialize')
+            .calls<Dash>(Dash, dash => dash.controlPlaneInitialize)
+
+        Route.get('/control-plane/upgrade')
+            .calls<Dash>(Dash, dash => dash.controlPlaneUpgrade)
+
         Route.group('/cobalt/resource', () => {
             Route.get('/:key/configure')
                 .parameterMiddleware(parseKey)

src/app/migrations/2023-04-01T03:28:47.961Z_CreateP5xNodesTableMigration.migration.ts

@@ -45,6 +45,10 @@ export default class CreateP5xNodesTableMigration extends Migration {
             .default(false)
             .required()
 
+        table.column('pve_host')
+            .type(FieldType.varchar)
+            .required()
+
         await schema.commit(table)
     }
 

src/app/models/HostGroup.model.ts

@@ -32,8 +32,8 @@ export class HostGroup extends Model<HostGroup> {
 
         // Insert the host records
         const rows = pveHosts.map(pveHost => ({
-            pveHost,
+            pve_host: pveHost,
-            hostGroupId: this.key(),
+            host_group_id: this.key(),
         }))
 
         await HostGroupHost.query<HostGroupHost>()
@@ -42,7 +42,8 @@ export class HostGroup extends Model<HostGroup> {
 
     public async toCobalt(): Promise<QueryRow> {
         const obj = this.toQueryRow()
-        obj.hosts = await this.hosts().get().then(x => x.pluck('pveHost'))
+        obj.hosts = await this.hosts().get().then(x => x.pluck('pveHost').all())
+        console.log('toCobalt', obj)
         return obj
     }
 }

src/app/models/IpRange.model.ts

@@ -47,7 +47,7 @@ export class IpRange extends Model<IpRange> {
             .filterOut(x => x.endsWith('.0'))
 
         // Look up already-allocated addresses
-        const allocatedIps = await Node.query<Node>().get().pluck('assignedIp')
+        const allocatedIps = await Node.query<Node>().get().map(x => x.assignedIp)
 
         // Find the first available address
         return possibleIps.firstWhere(ip => !allocatedIps.includes(ip))

src/app/models/Node.model.ts

@@ -5,6 +5,7 @@ import * as ssh2 from 'ssh2'
 import * as sshpk from 'sshpk'
 import {Setting} from './Setting.model'
 import {PCTHost} from '../support/hosts/PCTHost'
+import {Provisioner} from '../services/Provisioner.service'
 
 /**
  * Node Model
@@ -16,6 +17,19 @@ export class Node extends Model<Node> {
     protected static table = 'p5x_nodes'
     protected static key = 'node_id'
 
+    public static async getMaster(): Promise<Node> {
+        const master = await Node.query<Node>()
+            .where('is_permanent', '=', true)
+            .where('is_master', '=', true)
+            .first()
+
+        if ( !master ) {
+            throw new Error('Unable to get master node!')
+        }
+
+        return master
+    }
+
     @Field(FieldType.serial)
     public id?: number
 
@@ -37,6 +51,9 @@ export class Node extends Model<Node> {
     @Field(FieldType.bool, 'is_master')
     public isMaster: boolean = false
 
+    @Field(FieldType.varchar, 'pve_host')
+    public pveHost!: string
+
     public async getHost(): Promise<Host> {
         // Try to make a direct SSH connection to the container
         const privKey = await Setting.loadOneRequired('sshPrivateKey')
@@ -52,8 +69,15 @@ export class Node extends Model<Node> {
         }
 
         // Otherwise, fall back to the PCTHost proxy
+        const proxmox = await this.container().make<Provisioner>(Provisioner).getApi()
+        const ifaces = await proxmox.nodes.$(this.pveHost.split('/')[1]).network.$get()
+        const hostAddr = ifaces.filter(x => x.address)[0]?.address
+        if ( !hostAddr ) {
+            throw new Error('Unable to determine real address for host: ' + this.pveHost)
+        }
+
         return this.container().makeNew<Host>(PCTHost, {
-            host: await Setting.loadOneRequired('pveApiHost'),
+            host: hostAddr,
             username: 'root',
             password: await Setting.loadOneRequired('pveRootPassword'),
         }, this.pveId)

src/app/models/Setting.model.ts

@@ -38,7 +38,7 @@ export class Setting<Key extends keyof Settings> extends Model<Setting<Key>> {
 
     public static async set<Key extends keyof Settings>(key: Key, value: Settings[Key]): Promise<void> {
         // Delete any existing records w/ this key
-        await Setting.query().where('settingKey', '=', key).delete()
+        await Setting.query().where('setting_key', '=', key).delete()
 
         // Insert a new one
         const setting = Application.getContainer().makeNew<Setting<Key>>(Setting);
@@ -50,8 +50,8 @@ export class Setting<Key extends keyof Settings> extends Model<Setting<Key>> {
     public static async add<Key extends keyof Settings>(key: Key, value: Settings[Key]): Promise<void> {
         // Delete any existing records w/ this key and value
         await Setting.query()
-            .where('settingKey', '=', key)
+            .where('setting_key', '=', key)
-            .where('settingValue', '=', JSON.stringify(value))
+            .where('setting_value', '=', JSON.stringify(value))
             .delete()
 
         // Insert a new one

src/app/resources/assets/cobalt/component/Form.component.js

@@ -60,6 +60,15 @@ const template = `
                     :placeholder="field.placeholder || ''"
                     :readonly="mode === 'view' || field.readonly"
                 >
+                <select
+                    class="form-control"
+                    size="10"
+                    v-if="field.type === 'select'"
+                    :multiple="!!field.multiple"
+                    v-model="data[field.key]"
+                >
+                    <option v-for="option of field.options" :value="option.value">{{ option.display }}</option>
+                </select>
                 <cobalt-monaco
                     v-if="field.type === 'html' && !(mode === 'view' || field.readonly)"
                     syntax="html"

src/app/resources/views/dash/control-plane.pug

@@ -0,0 +1,22 @@
+extends template
+
+block content
+    h1 Control Plane!
+    if !hasKubelet
+        p The control plane has not yet been initialized on this cluster.
+        a.btn.btn-primary.btn-icon-split(href=route('/dash/control-plane/initialize'))
+            span.icon.text-white-50
+                i.fas.fa-server
+            span.text Initialize Master Node
+    else
+        p
+            b Client version: 
+            span #{version.clientVersion.major}.#{version.clientVersion.minor} (#{version.clientVersion.gitVersion})
+            span    |   
+            b Server version: 
+            span #{version.serverVersion.major}.#{version.serverVersion.minor} (#{version.serverVersion.gitVersion})
+        p
+            a.btn.btn-warning.btn-icon-split(href=route('/dash/control-plane/upgrade'))
+                span.icon.text-white-50
+                    i.fas.fa-upload
+                span.text Upgrade Master Node

src/app/resources/views/dash/template.pug

@@ -11,6 +11,7 @@ head
         link(href='https://fonts.googleapis.com/css?family=Nunito:200,200i,300,300i,400,400i,600,600i,700,700i,800,800i,900,900i' rel='stylesheet')
         link(href=asset('dash/css/sb-admin-2.min.css') rel='stylesheet')
         link(href=asset('dash/vendor/fontawesome-free/css/all.min.css') rel='stylesheet' type='text/css')
+        //link(href=asset('md5/css/mdb.min.css') rel='stylesheet' type='text/css')
         link(href=asset('dash/vendor/datatables2/dataTables.bootstrap4.min.css'))
         link(rel='stylesheet' href='https://unpkg.com/vue2-datepicker@latest/index.css')
         link(rel='stylesheet' data-name='vs/editor/editor.main' href=asset('monaco/package/min/vs/editor/editor.main.css'))
@@ -21,6 +22,10 @@ head
                 | #{config('app.name')}
         hr.sidebar-divider
         .sidebar-heading Cluster
+        li.nav-item
+            a.nav-link(href=route('dash/control-plane'))
+                i.fas.fa-fw.fa-server
+                span Control Plane
         li.nav-item
             a.nav-link(href=route('dash/cobalt/listing/hostgroup'))
                 i.fas.fa-fw.fa-server
@@ -111,7 +116,8 @@ a.scroll-to-top.rounded(href='#page-top')
 
 block scripts
     script(src=asset('dash/vendor/jquery/jquery.min.js'))
-    script(src=asset('dash/vendor/bootstrap/js/bootstrap.bundle.min.js'))
+    //script(src=asset('dash/vendor/bootstrap/js/bootstrap.bundle.min.js'))
+    //script(src=asset('md5/js/mdb.min.js'))
     script(src=asset('dash/vendor/jquery-easing/jquery.easing.min.js'))
     script(src=asset('vue.js'))
     script(src=asset('vues6.js') type='module')

src/app/services/Images.service.ts

@@ -11,7 +11,7 @@ export interface LXCImage {
 
 @Singleton()
 export class Images {
-    private imageServer = 'https://us.lxd.images.canonical.com'
+    private imageServer = 'https://images.linuxcontainers.org'
     private listEndpoint = 'meta/1.0/index-system'
     private imageBinaryName = 'rootfs.tar.xz'
 

src/app/services/Provisioner.service.ts

@@ -6,7 +6,7 @@ import {
     fetch,
     Container,
     Logging,
-    universalPath, uuid4,
+    universalPath, uuid4, collect, ArrayElement, Collection, Maybe,
 } from '@extollo/lib'
 import { Node } from "../models/Node.model";
 import {Setting} from '../models/Setting.model'
@@ -17,6 +17,14 @@ import {unsafeESMImport} from '@extollo/lib/lib/util/unsafe'
 import * as sshpk from 'sshpk'
 import {shellCommand, SSHHost} from '../support/hosts'
 import {User} from '../models/User.model'
+import {HostGroupHost} from '../models/HostGroupHost.model'
+import {HostGroup} from '../models/HostGroup.model'
+
+export interface HostUsage {
+    host: HostGroupHost,
+    mem: { total: number, used: number },
+    cpu: { total: number, used: number },
+}
 
 @Singleton()
 export class Provisioner {
@@ -31,6 +39,188 @@ export class Provisioner {
 
     private cachedProxmoxApi?: Proxmox.Api
 
+    public async selectCandidate(group: HostGroup): Promise<Maybe<HostGroupHost>> {
+        return (await this.getNodeResources(group))
+            .sortByDesc(u => ((u.mem.used / u.mem.total) + (u.cpu.used / u.cpu.total)) / 2)
+            .first()
+            ?.host
+    }
+
+    private async getNodeResources(group: HostGroup): Promise<Collection<HostUsage>> {
+        // Get the Proxmox API
+        const proxmox = await this.getApi()
+
+        // Look up all host group hosts
+        const hosts = await group.hosts().get()
+
+        // Query proxmox to identify the available resource on each node
+        return hosts.promiseMap(async host => {
+            const containers = await proxmox.nodes.$(host.pveHost.split('/')[1]).lxc.$get()
+            const vms = await proxmox.nodes.$(host.pveHost.split('/')[1]).qemu.$get()
+            const status = await proxmox.nodes.$(host.pveHost.split('/')[1]).status.$get()
+
+            const totalMem = status.memory.free
+            const totalCpu = status.cpuinfo.cpus
+
+            const containerMem = containers.length ? collect(containers).sum(x => x.maxmem) : 0
+            const vmMem = vms.length ? collect(vms).sum(x => x.maxmem) : 0
+
+            const containerCpu = containers.length ? collect(containers).sum(x => x.cpus) : 0
+            const vmCpu = vms.length ? collect(vms).sum(x => x.cpus) : 0
+
+            return {
+                host,
+                mem: { total: totalMem, used: containerMem + vmMem },
+                cpu: { total: totalCpu, used: containerCpu + vmCpu },
+            }
+        })
+    }
+
+    private async cacheBaseImage(host: HostGroupHost): Promise<void> {
+        // Get the Proxmox API!
+        const proxmox = await this.getApi()
+        const pveHost = proxmox.nodes.$(host!.pveHost.split('/')[1])
+        const imageStorage = pveHost.storage.$('p5x.image-cache')
+
+        // Look up the appropriate LXC image
+        this.logging.info('Searching for base LXC template image...')
+        const image = await this.images.getImages()
+            .then(x =>
+                x.where('os', '=', 'ubuntu')
+                    .where('version', '=', 'jammy')  // 22.04 LTS
+                    .where('format', '=', 'cloud')
+                    .where('arch', '=', 'amd64')
+                    .sortByDesc('timestamp')
+                    .first()
+            )
+
+        if ( !image ) {
+            throw new ErrorWithContext('Unable to find Ubuntu 22.04 LXC template binary')
+        }
+
+        const imageUrl = this.images.getImageBinaryPath(image!)
+
+        // Upload the template image to the storage mount
+        const uploadTaskUPID = await imageStorage['download-url'].$post({
+            content: 'vztmpl',
+            filename: 'p5x-base.tar.xz',
+            url: imageUrl.toRemote,
+        })
+
+        this.logging.info('Waiting for LXC image to upload to PVE node...')
+        await this.waitForNodeTask(host!.pveHost.split('/')[1], uploadTaskUPID)
+    }
+
+    private async getNextHostname(): Promise<string> {
+        const nextNodeNum = (await Setting.load('nextNodeNum')).first() || 0
+        await Setting.set('nextNodeNum', nextNodeNum + 1)
+        return `p5x-node-${(Math.random() + 1).toString(36).substring(9)}${nextNodeNum}`
+    }
+
+    public async provisionNode(group: HostGroup): Promise<Node> {
+        // Look up a target node for provisioning
+        const host = await this.selectCandidate(group)
+        if ( !host ) {
+            throw new Error('Unable to find host in host group for provisioning!')
+        }
+
+        // Get the Proxmox API!
+        const proxmox = await this.getApi()
+
+        // Look up the image cache on the node
+        const pveHost = proxmox.nodes.$(host!.pveHost.split('/')[1])
+        const imageStorage = pveHost.storage.$('p5x.image-cache')
+
+        // Check if the host needs to pull the image
+        const imageContent = await imageStorage.content.$get()
+        const hasBaseImage = imageContent.some(i =>
+            i.volid === 'p5x.image-cache:vztmpl/p5x-base.tar.xz')
+
+        if ( !hasBaseImage ) {
+            await this.cacheBaseImage(host)
+        }
+
+        // Look up the P5x storage pool from the settings
+        const pveStoragePool = await Setting.loadOneRequired('pveStoragePool')
+
+        // Look up the primary IP pool range
+        const ipRange = await IpRange.getDefault()
+
+        // Get the next vmid to be used by the LXC container
+        const nodeHostname = await this.getNextHostname()
+        const nodeVMID = await proxmox.cluster.nextid.$get()
+        const nodeIP = await ipRange.getNextAvailableOrFail()
+        const sshPubKey = sshpk.parseKey(await Setting.loadOneRequired('sshPublicKey'))
+            .toString('ssh')
+
+        // Create a new LXC container on the node for the new instance
+        const nodeTaskUPID = await pveHost.lxc.$post({
+            ostemplate: 'p5x.image-cache:vztmpl/p5x-base.tar.xz',
+            vmid: nodeVMID,
+            cores: await Setting.loadOneRequired('nodeCpus'),
+            description: 'P5x Worker Node',
+            hostname: nodeHostname,
+            memory: await Setting.loadOneRequired('nodeRamMib'),
+            net0:
+                objectToKeyValue({
+                    name: 'eth0',
+                    bridge: await Setting.loadOneRequired('nodeNetworkBridge'),
+                    firewall: 1,
+                    gw: ipRange.gatewayIp,
+                    ip: `${nodeIP}/${ipRange.subnet}`,
+                })
+                    .map(x => `${x.key}=${x.value}`)
+                    .join(','),
+            onboot: true,
+            password: 'strongpassword',  //rootPassword,  // fixme
+            rootfs: `${pveStoragePool}:8`,  // 8 GiB  // fixme
+            searchdomain: await Setting.loadOneRequired('dnsDomain'),
+            'ssh-public-keys': sshPubKey,
+            start: true,
+            storage: pveStoragePool,
+            tags: 'p5x',
+        })
+
+        this.logging.info('Waiting for PVE node to be created...')
+        await this.waitForNodeTask(host!.pveHost.split('/')[1], nodeTaskUPID)
+
+        // Save the node
+        const node = this.container.makeNew<Node>(Node)
+        node.pveId = nodeVMID
+        node.hostname = nodeHostname
+        node.pveHost = host!.pveHost
+        node.assignedIp = nodeIP
+        node.assignedSubnet = ipRange.subnet
+        node.isPermanent = false
+        node.isMaster = false
+        await node.save()
+
+        // Wait for the host to come online (this will be a PCTHost passed through PVE)
+        this.logging.info('Waiting for node to come online...')
+        const proxyHost = await node.getHost()
+        await proxyHost.waitForAlive()
+
+        // Configure SSH on the instance
+        this.logging.info('Configuring SSH access to node...')
+        await proxyHost.run(shellCommand('apt update'))
+        await proxyHost.run(shellCommand('apt install -y openssh-server curl'))
+        await proxyHost.run(shellCommand('systemctl enable --now ssh'))
+        await new Promise<void>(res => setTimeout(() => res(), 5000))
+
+        // Try to re-connect to the instance directly
+        this.logging.info('Attempting to re-connect to node via SSH...')
+        const sshHost = await node.getHost()
+        if ( !(sshHost instanceof SSHHost) ) {
+            throw new ErrorWithContext('Failed to establish direct connection to node', {
+                node,
+                proxyHost,
+                sshHost,
+            })
+        }
+
+        return node
+    }
+
     public async provisionMasterNode(): Promise<Node> {
         // Look up the name of the master node from the settings
         const pveMasterNode = await Setting.loadOneRequired('pveMasterNode')
@@ -79,7 +269,7 @@ export class Provisioner {
             )
 
         if ( !image ) {
-            throw new ErrorWithContext('Unable to find Debian Bookworm LXC template binary')
+            throw new ErrorWithContext('Unable to find Ubuntu 22.04 LXC template binary')
         }
 
         const imageUrl = this.images.getImageBinaryPath(image!)
@@ -136,6 +326,7 @@ export class Provisioner {
         const node = this.container.makeNew<Node>(Node)
         node.pveId = masterVMID
         node.hostname = 'p5x-control'
+        node.pveHost = `node/${pveMasterNode}`
         node.assignedIp = masterIP
         node.assignedSubnet = ipRange.subnet
         node.isPermanent = true

src/app/support/hosts/Host.ts

@@ -1,8 +1,8 @@
 import {
     Awaitable,
     AwareOfContainerLifecycle, ErrorWithContext,
-    Filesystem,
+    Filesystem, hasOwnProperty,
-    infer,
+    infer, Maybe,
     objectToKeyValue,
     UniversalPath,
     uuid4,
@@ -12,6 +12,50 @@ import {ExecutionResult} from './ExecutionResult'
 import {CommandError} from './errors'
 import {SystemMetrics} from './SystemMetrics'
 
+export interface SystemDUnitDetails {
+    Type: string
+    Result: 'success'|string
+    NRestarts: number
+    Id: string
+    Names: string[]
+    Requires?: string[]
+    Wants?: string[]
+    WantedBy?: string
+    Description?: string
+    ActiveState?: 'active'|string
+}
+
+export const isSystemDUnitDetails = (what: unknown): what is SystemDUnitDetails => {
+    return !!(
+        typeof what === 'object'
+        && what
+        && hasOwnProperty(what, 'Type') && typeof what.Type === 'string'
+        && hasOwnProperty(what, 'Result') && typeof what.Result === 'string'
+        && hasOwnProperty(what, 'NRestarts') && typeof what.NRestarts === 'number'
+        && hasOwnProperty(what, 'Id') && typeof what.Id === 'string'
+        && (
+            (hasOwnProperty(what, 'Requires') && Array.isArray(what.Requires) && what.Requires.every(x => typeof x === 'string'))
+            || !hasOwnProperty(what, 'Requires')
+        )
+        && (
+            (hasOwnProperty(what, 'Wants') && Array.isArray(what.Wants) && what.Wants.every(x => typeof x === 'string'))
+            || !hasOwnProperty(what, 'Wants')
+        )
+        && (
+            (hasOwnProperty(what, 'WantedBy') && typeof what.WantedBy === 'string')
+            || !hasOwnProperty(what, 'WantedBy')
+        )
+        && (
+            (hasOwnProperty(what, 'Description') && typeof what.Description === 'string')
+            || !hasOwnProperty(what, 'Description')
+        )
+        && (
+            (hasOwnProperty(what, 'ActiveState') && typeof what.ActiveState === 'string')
+            || !hasOwnProperty(what, 'ActiveState')
+        )
+    )
+}
+
 export class StandardCommandPalette implements CommandPalette {
     changeDirectory = shellCommand(`cd "%%PATH%%"`)
     cpuPercentage = shellCommand(`grep 'cpu ' /proc/stat | awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage}'`)
@@ -30,6 +74,7 @@ export class StandardCommandPalette implements CommandPalette {
     tempFile = shellCommand(`mktemp`)
     tempPath = shellCommand(`mktemp -d`)
     echo = shellCommand(`echo "%%OUTPUT%%"`)
+    systemDUnitShow = shellCommand(`systemctl show --no-pager %%UNIT%%`)
 
     public format(command: keyof CommandPaletteCommands, substitutions: Record<string, any>): ShellCommand {
         let formatted = `${this[command]}`
@@ -200,4 +245,34 @@ export abstract class Host implements AwareOfContainerLifecycle {
     public reboot(): Awaitable<ExecutionResult> {
         return this.run(this.getCommandPalette().format('reboot', {}))
     }
+
+    public async getSystemDUnit(unit: string): Promise<Maybe<SystemDUnitDetails>> {
+        const cmd = this.getCommandPalette().format('systemDUnitShow', { unit })
+        const result = await this.run(cmd)
+        const listing = result.standardOutput.join('\n')
+            .trim()
+            .split('\n')
+            .map(x => ({
+                key: x.split('=')[0],
+                value: x.split('=').slice(1).join('='),
+            }))
+            .map((x: {key: string, value: any}) => {
+                if ( ['Names', 'Requires', 'Wants'].includes(x.key) ) {
+                    x.value = x.value.split(' ').filter(Boolean)
+                } else {
+                    x.value = infer(x.value)
+                }
+
+                return {[x.key]: x.value}
+            })
+            .reduce((a, b) => ({...a, ...b}))
+
+        if ( hasOwnProperty(listing, 'LoadError') ) {
+            return
+        }
+
+        if ( isSystemDUnitDetails(listing) ) {
+            return listing
+        }
+    }
 }

src/app/support/hosts/types.ts

@@ -22,6 +22,7 @@ export interface CommandPaletteCommands {
     fileDirectoryOwnershipSetFlat: ShellCommand
     fileDirectoryOwnershipSetRecursive: ShellCommand
     echo: ShellCommand
+    systemDUnitShow: ShellCommand
 }
 
 export type CommandPalette = CommandPaletteCommands & {

src/app/support/k8s/ControlPlane.ts

@@ -0,0 +1,88 @@
+import {Host, ShellCommand, shellCommand} from '../hosts'
+import {hasOwnProperty} from '@extollo/lib'
+import {Node} from '../../models/Node.model'
+
+
+interface ControlPlaneSideVersion {
+	major: string,
+	minor: string,
+	gitVersion: string,
+	goVersion: string,
+}
+
+const isControlPlaneSideVersion = (what: unknown): what is ControlPlaneSideVersion => {
+	return !!(
+		typeof what === 'object'
+		&& what
+		&& hasOwnProperty(what, 'major') && typeof what.major === 'string'
+		&& hasOwnProperty(what, 'minor') && typeof what.minor === 'string'
+		&& hasOwnProperty(what, 'gitVersion') && typeof what.gitVersion === 'string'
+		&& hasOwnProperty(what, 'goVersion') && typeof what.goVersion === 'string'
+	)
+}
+
+export interface ControlPlaneVersion {
+	clientVersion: ControlPlaneSideVersion,
+	serverVersion: ControlPlaneSideVersion,
+}
+
+export const isControlPlaneVersion = (what: unknown): what is ControlPlaneVersion => {
+	return !!(
+		typeof what === 'object'
+		&& what
+		&& hasOwnProperty(what, 'clientVersion') && isControlPlaneSideVersion(what.clientVersion)
+		&& hasOwnProperty(what, 'serverVersion') && isControlPlaneSideVersion(what.serverVersion)
+	)
+}
+
+export class ControlPlane {
+	constructor(
+		private host: Host
+	) {}
+
+	public async hasKubelet(): Promise<boolean> {
+		return !!(await this.host.getSystemDUnit('k3s'))
+	}
+
+	public async getVersion(node?: Node): Promise<ControlPlaneVersion> {
+		const host = node ? (await node.getHost()) : this.host
+		const result = await host.run(shellCommand('kubectl version --output=json'))
+		const version = JSON.parse(result.standardOutput.join('\n'))
+		if ( !isControlPlaneVersion(version) ) {
+			throw new Error('Invalid response from control plane: ' + result.standardOutput.join('\n'))
+		}
+
+		return version
+	}
+
+	public async initializeMaster(): Promise<ControlPlaneVersion> {
+		await this.host.run(shellCommand('apt-get update'))
+		await this.host.run(shellCommand('apt-get install -y curl'))
+		await this.host.run(await this.getK3sMasterCommand())
+		return this.getVersion()
+	}
+
+	public async initializeNode(node: Node): Promise<ControlPlaneVersion> {
+		const host = await node.getHost()
+		await host.run(shellCommand('apt-get update'))
+		await host.run(shellCommand('apt-get install -y curl'))
+		await host.run(await this.getK3sNodeCommand(node))
+		return this.getVersion(node)
+	}
+
+	private async getK3sNodeCommand(node: Node): Promise<ShellCommand> {
+		const master = await Node.getMaster()
+		const token = await this.getMasterToken()
+		return shellCommand(`curl -fsL https://get.k3s.io | K3S_URL=https://${master.assignedIp}:6443 K3S_TOKEN=${token} sh -s - --node-name ${node.hostname}`)
+	}
+
+	private async getMasterToken(): Promise<string> {
+		const result = await this.host.run(shellCommand('cat /var/lib/rancher/k3s/server/node-token'))
+		return result.standardOutput.join('\n').trim()
+	}
+
+	private async getK3sMasterCommand(): Promise<ShellCommand> {
+		const master = await Node.getMaster()
+		return shellCommand(`curl -fsL https://get.k3s.io | sh -s - --disable traefik --node-name ${master.hostname}`)
+	}
+}

src/app/types.ts

@@ -28,4 +28,5 @@ export type Settings = Record<string, JSONState> & {
     nodeNetworkBridge: string
     nodeCpus: number
     nodeRamMib: number
+    nextNodeNum: number
 }