[WIP] Early provisioning implementation
This commit is contained in:
parent
17fda7b6ef
commit
d3398f82d6
@ -62,7 +62,7 @@ export class StandupDirective extends Directive {
|
||||
ipRange.endIp = this.getIpOptionOrFail('ip-pool-end')
|
||||
ipRange.subnet = this.getSubnetOptionOrFail('ip-pool-subnet')
|
||||
ipRange.gatewayIp = this.getIpOptionOrFail('ip-pool-gateway')
|
||||
ipRange.save()
|
||||
await ipRange.save()
|
||||
|
||||
// Provision the master node:
|
||||
const masterNode = await this.provisioner.provisionMasterNode()
|
||||
|
@ -70,7 +70,7 @@ export type RemoteSelectDefinition = {
|
||||
}
|
||||
|
||||
export type FieldDefinition = FieldBase
|
||||
| FieldBase & { type: FieldType.select, options: SelectOptions }
|
||||
| FieldBase & { type: FieldType.select, multiple?: boolean, options: SelectOptions }
|
||||
| FieldBase & { type: FieldType.select } & RemoteSelectDefinition
|
||||
|
||||
export interface DataSourceController {
|
||||
|
@ -42,6 +42,7 @@ export default {
|
||||
display: 'Physical Hosts',
|
||||
type: FieldType.select,
|
||||
required: true,
|
||||
multiple: true,
|
||||
displayFrom: 'pveDisplay',
|
||||
valueFrom: 'pveHost',
|
||||
hideOn: {
|
||||
|
@ -2,21 +2,72 @@ import {
|
||||
Controller,
|
||||
Injectable,
|
||||
Inject,
|
||||
Logging, ResponseObject, view,
|
||||
Logging, ResponseObject, view, redirect,
|
||||
} from '@extollo/lib'
|
||||
import {ControlPlane} from '../../support/k8s/ControlPlane'
|
||||
import {Node} from '../../models/Node.model'
|
||||
import {Provisioner} from '../../services/Provisioner.service'
|
||||
import {HostGroup} from '../../models/HostGroup.model'
|
||||
|
||||
@Injectable()
|
||||
export class Dash extends Controller {
|
||||
@Inject()
|
||||
protected readonly logging!: Logging
|
||||
|
||||
@Inject()
|
||||
protected readonly provisioner!: Provisioner
|
||||
|
||||
public main(): ResponseObject {
|
||||
return view('dash:template')
|
||||
}
|
||||
|
||||
public provision(): ResponseObject {
|
||||
public async provision(): Promise<ResponseObject> {
|
||||
const cd = await this.provisioner.provisionNode((await HostGroup.findByKey(2))!)
|
||||
const cp = new ControlPlane(await Node.getMaster().then(m => m.getHost()))
|
||||
const cv = await cp.initializeNode(cd)
|
||||
console.log({cv})
|
||||
return view('dash:provision', {
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
public async controlPlane(): Promise<ResponseObject> {
|
||||
const master = await Node.getMaster()
|
||||
const plane = new ControlPlane(await master.getHost())
|
||||
|
||||
const hasKubelet = await plane.hasKubelet()
|
||||
const extras: any = {}
|
||||
if ( hasKubelet ) {
|
||||
extras.version = await plane.getVersion()
|
||||
}
|
||||
|
||||
return view('dash:control-plane', {
|
||||
hasKubelet,
|
||||
...extras,
|
||||
})
|
||||
}
|
||||
|
||||
public async controlPlaneInitialize(): Promise<ResponseObject> {
|
||||
const master = await Node.getMaster()
|
||||
const plane = new ControlPlane(await master.getHost())
|
||||
|
||||
if ( await plane.hasKubelet() ) {
|
||||
return redirect('/dash/control-plane')
|
||||
}
|
||||
|
||||
await plane.initializeMaster()
|
||||
return redirect('/dash/control-plane')
|
||||
}
|
||||
|
||||
public async controlPlaneUpgrade(): Promise<ResponseObject> {
|
||||
const master = await Node.getMaster()
|
||||
const plane = new ControlPlane(await master.getHost())
|
||||
|
||||
if ( await plane.hasKubelet() ) {
|
||||
await plane.initializeMaster()
|
||||
return redirect('/dash/control-plane')
|
||||
}
|
||||
|
||||
return redirect('/dash/control-plane')
|
||||
}
|
||||
}
|
||||
|
@ -10,7 +10,7 @@ import {
|
||||
HTTPError,
|
||||
HTTPStatus,
|
||||
Inject,
|
||||
Injectable, Iterable,
|
||||
Injectable, Iterable, JSONState,
|
||||
Maybe,
|
||||
QueryRow,
|
||||
} from '@extollo/lib'
|
||||
@ -273,7 +273,7 @@ export class ResourceAPI extends Controller {
|
||||
throw new HTTPError(HTTPStatus.BAD_REQUEST, `Missing required field: ${key}`)
|
||||
}
|
||||
|
||||
if ( isNaN(cast) ) {
|
||||
if ( value && value !== 0 && isNaN(cast) ) {
|
||||
throw new HTTPError(HTTPStatus.BAD_REQUEST, `Field must be a number: ${key}`)
|
||||
}
|
||||
return cast
|
||||
@ -283,7 +283,7 @@ export class ResourceAPI extends Controller {
|
||||
throw new HTTPError(HTTPStatus.BAD_REQUEST, `Missing required field: ${key}`)
|
||||
}
|
||||
|
||||
if ( isNaN(cast) ) {
|
||||
if ( value && value !== 0 && isNaN(cast) ) {
|
||||
throw new HTTPError(HTTPStatus.BAD_REQUEST, `Field must be a number: ${key}`)
|
||||
}
|
||||
return cast
|
||||
@ -308,8 +308,10 @@ export class ResourceAPI extends Controller {
|
||||
return cast
|
||||
} else if ( type === FieldType.select && hasOwnProperty(fieldDef, 'options') ) {
|
||||
const options = collect(fieldDef.options as SelectOptions)
|
||||
if ( options.pluck('value').includes(value) ) {
|
||||
return value
|
||||
const selectedValues = Array.isArray(value) ? value : [value]
|
||||
const validOptions = options.pluck('value').intersect(selectedValues)
|
||||
if ( validOptions.isNotEmpty() ) {
|
||||
return fieldDef.multiple ? validOptions.all() : validOptions.get(0)
|
||||
}
|
||||
|
||||
if ( required ) {
|
||||
|
@ -27,7 +27,7 @@ export class HostGroups extends Controller implements DataSourceController {
|
||||
insert(row: QueryRow): Awaitable<QueryRow> {
|
||||
const hosts = row.hosts
|
||||
if ( !Array.isArray(hosts) || !hosts.every(i => typeof i === 'string') ) {
|
||||
throw new Error('Invalid hosts: must be number[]')
|
||||
throw new Error('Invalid hosts: must be string[]')
|
||||
}
|
||||
|
||||
return AsyncPipe.wrap(this.request.makeNew<HostGroup>(HostGroup))
|
||||
@ -59,7 +59,7 @@ export class HostGroups extends Controller implements DataSourceController {
|
||||
}
|
||||
|
||||
async delete(id: any): Promise<void> {
|
||||
HostGroup.query<HostGroup>()
|
||||
await HostGroup.query<HostGroup>()
|
||||
.whereKey(id)
|
||||
.delete()
|
||||
}
|
||||
@ -69,10 +69,12 @@ export class HostGroups extends Controller implements DataSourceController {
|
||||
.tap(p => p.getApi())
|
||||
.tap(api => api.nodes.$get())
|
||||
.tap(nodes =>
|
||||
nodes.map(node => ({
|
||||
pveHost: node.id || node.node,
|
||||
pveDisplay: `${node.node} (CPUs: ${node.maxcpu}, RAM: ${Math.floor((node.maxmem || 0) / 1000000000)})`
|
||||
}))
|
||||
nodes
|
||||
.map(node => ({
|
||||
pveHost: node.id || node.node,
|
||||
pveDisplay: `${node.node} (CPUs: ${node.maxcpu}, RAM: ${Math.floor((node.maxmem || 0) / 1000000000)})`
|
||||
}))
|
||||
.sort((a, b) => a.pveDisplay.localeCompare(b.pveDisplay))
|
||||
)
|
||||
.resolve()
|
||||
}
|
||||
|
@ -33,6 +33,15 @@ Route
|
||||
Route.get('/provision')
|
||||
.calls<Dash>(Dash, dash => dash.provision)
|
||||
|
||||
Route.get('/control-plane')
|
||||
.calls<Dash>(Dash, dash => dash.controlPlane)
|
||||
|
||||
Route.get('/control-plane/initialize')
|
||||
.calls<Dash>(Dash, dash => dash.controlPlaneInitialize)
|
||||
|
||||
Route.get('/control-plane/upgrade')
|
||||
.calls<Dash>(Dash, dash => dash.controlPlaneUpgrade)
|
||||
|
||||
Route.group('/cobalt/resource', () => {
|
||||
Route.get('/:key/configure')
|
||||
.parameterMiddleware(parseKey)
|
||||
|
@ -45,6 +45,10 @@ export default class CreateP5xNodesTableMigration extends Migration {
|
||||
.default(false)
|
||||
.required()
|
||||
|
||||
table.column('pve_host')
|
||||
.type(FieldType.varchar)
|
||||
.required()
|
||||
|
||||
await schema.commit(table)
|
||||
}
|
||||
|
||||
|
@ -32,8 +32,8 @@ export class HostGroup extends Model<HostGroup> {
|
||||
|
||||
// Insert the host records
|
||||
const rows = pveHosts.map(pveHost => ({
|
||||
pveHost,
|
||||
hostGroupId: this.key(),
|
||||
pve_host: pveHost,
|
||||
host_group_id: this.key(),
|
||||
}))
|
||||
|
||||
await HostGroupHost.query<HostGroupHost>()
|
||||
@ -42,7 +42,8 @@ export class HostGroup extends Model<HostGroup> {
|
||||
|
||||
public async toCobalt(): Promise<QueryRow> {
|
||||
const obj = this.toQueryRow()
|
||||
obj.hosts = await this.hosts().get().then(x => x.pluck('pveHost'))
|
||||
obj.hosts = await this.hosts().get().then(x => x.pluck('pveHost').all())
|
||||
console.log('toCobalt', obj)
|
||||
return obj
|
||||
}
|
||||
}
|
||||
|
@ -47,7 +47,7 @@ export class IpRange extends Model<IpRange> {
|
||||
.filterOut(x => x.endsWith('.0'))
|
||||
|
||||
// Look up already-allocated addresses
|
||||
const allocatedIps = await Node.query<Node>().get().pluck('assignedIp')
|
||||
const allocatedIps = await Node.query<Node>().get().map(x => x.assignedIp)
|
||||
|
||||
// Find the first available address
|
||||
return possibleIps.firstWhere(ip => !allocatedIps.includes(ip))
|
||||
|
@ -5,6 +5,7 @@ import * as ssh2 from 'ssh2'
|
||||
import * as sshpk from 'sshpk'
|
||||
import {Setting} from './Setting.model'
|
||||
import {PCTHost} from '../support/hosts/PCTHost'
|
||||
import {Provisioner} from '../services/Provisioner.service'
|
||||
|
||||
/**
|
||||
* Node Model
|
||||
@ -16,6 +17,19 @@ export class Node extends Model<Node> {
|
||||
protected static table = 'p5x_nodes'
|
||||
protected static key = 'node_id'
|
||||
|
||||
public static async getMaster(): Promise<Node> {
|
||||
const master = await Node.query<Node>()
|
||||
.where('is_permanent', '=', true)
|
||||
.where('is_master', '=', true)
|
||||
.first()
|
||||
|
||||
if ( !master ) {
|
||||
throw new Error('Unable to get master node!')
|
||||
}
|
||||
|
||||
return master
|
||||
}
|
||||
|
||||
@Field(FieldType.serial)
|
||||
public id?: number
|
||||
|
||||
@ -37,6 +51,9 @@ export class Node extends Model<Node> {
|
||||
@Field(FieldType.bool, 'is_master')
|
||||
public isMaster: boolean = false
|
||||
|
||||
@Field(FieldType.varchar, 'pve_host')
|
||||
public pveHost!: string
|
||||
|
||||
public async getHost(): Promise<Host> {
|
||||
// Try to make a direct SSH connection to the container
|
||||
const privKey = await Setting.loadOneRequired('sshPrivateKey')
|
||||
@ -52,8 +69,15 @@ export class Node extends Model<Node> {
|
||||
}
|
||||
|
||||
// Otherwise, fall back to the PCTHost proxy
|
||||
const proxmox = await this.container().make<Provisioner>(Provisioner).getApi()
|
||||
const ifaces = await proxmox.nodes.$(this.pveHost.split('/')[1]).network.$get()
|
||||
const hostAddr = ifaces.filter(x => x.address)[0]?.address
|
||||
if ( !hostAddr ) {
|
||||
throw new Error('Unable to determine real address for host: ' + this.pveHost)
|
||||
}
|
||||
|
||||
return this.container().makeNew<Host>(PCTHost, {
|
||||
host: await Setting.loadOneRequired('pveApiHost'),
|
||||
host: hostAddr,
|
||||
username: 'root',
|
||||
password: await Setting.loadOneRequired('pveRootPassword'),
|
||||
}, this.pveId)
|
||||
|
@ -38,7 +38,7 @@ export class Setting<Key extends keyof Settings> extends Model<Setting<Key>> {
|
||||
|
||||
public static async set<Key extends keyof Settings>(key: Key, value: Settings[Key]): Promise<void> {
|
||||
// Delete any existing records w/ this key
|
||||
await Setting.query().where('settingKey', '=', key).delete()
|
||||
await Setting.query().where('setting_key', '=', key).delete()
|
||||
|
||||
// Insert a new one
|
||||
const setting = Application.getContainer().makeNew<Setting<Key>>(Setting);
|
||||
@ -50,8 +50,8 @@ export class Setting<Key extends keyof Settings> extends Model<Setting<Key>> {
|
||||
public static async add<Key extends keyof Settings>(key: Key, value: Settings[Key]): Promise<void> {
|
||||
// Delete any existing records w/ this key and value
|
||||
await Setting.query()
|
||||
.where('settingKey', '=', key)
|
||||
.where('settingValue', '=', JSON.stringify(value))
|
||||
.where('setting_key', '=', key)
|
||||
.where('setting_value', '=', JSON.stringify(value))
|
||||
.delete()
|
||||
|
||||
// Insert a new one
|
||||
|
@ -60,6 +60,15 @@ const template = `
|
||||
:placeholder="field.placeholder || ''"
|
||||
:readonly="mode === 'view' || field.readonly"
|
||||
>
|
||||
<select
|
||||
class="form-control"
|
||||
size="10"
|
||||
v-if="field.type === 'select'"
|
||||
:multiple="!!field.multiple"
|
||||
v-model="data[field.key]"
|
||||
>
|
||||
<option v-for="option of field.options" :value="option.value">{{ option.display }}</option>
|
||||
</select>
|
||||
<cobalt-monaco
|
||||
v-if="field.type === 'html' && !(mode === 'view' || field.readonly)"
|
||||
syntax="html"
|
||||
|
23
src/app/resources/assets/md5/css/mdb.dark.min.css
vendored
Normal file
23
src/app/resources/assets/md5/css/mdb.dark.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
1
src/app/resources/assets/md5/css/mdb.dark.min.css.map
Normal file
1
src/app/resources/assets/md5/css/mdb.dark.min.css.map
Normal file
File diff suppressed because one or more lines are too long
10
src/app/resources/assets/md5/css/mdb.dark.rtl.min.css
vendored
Normal file
10
src/app/resources/assets/md5/css/mdb.dark.rtl.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
40
src/app/resources/assets/md5/css/mdb.min.css
vendored
Normal file
40
src/app/resources/assets/md5/css/mdb.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
1
src/app/resources/assets/md5/css/mdb.min.css.map
Normal file
1
src/app/resources/assets/md5/css/mdb.min.css.map
Normal file
File diff suppressed because one or more lines are too long
10
src/app/resources/assets/md5/css/mdb.rtl.min.css
vendored
Normal file
10
src/app/resources/assets/md5/css/mdb.rtl.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
1
src/app/resources/assets/md5/css/mdb.rtl.min.css.map
Normal file
1
src/app/resources/assets/md5/css/mdb.rtl.min.css.map
Normal file
File diff suppressed because one or more lines are too long
BIN
src/app/resources/assets/md5/img/mdb-favicon.ico
Normal file
BIN
src/app/resources/assets/md5/img/mdb-favicon.ico
Normal file
Binary file not shown.
After (image error) Size: 1.1 KiB |
20
src/app/resources/assets/md5/js/mdb.min.js
vendored
Normal file
20
src/app/resources/assets/md5/js/mdb.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
1
src/app/resources/assets/md5/js/mdb.min.js.map
Normal file
1
src/app/resources/assets/md5/js/mdb.min.js.map
Normal file
File diff suppressed because one or more lines are too long
22
src/app/resources/views/dash/control-plane.pug
Normal file
22
src/app/resources/views/dash/control-plane.pug
Normal file
@ -0,0 +1,22 @@
|
||||
extends template
|
||||
|
||||
block content
|
||||
h1 Control Plane!
|
||||
if !hasKubelet
|
||||
p The control plane has not yet been initialized on this cluster.
|
||||
a.btn.btn-primary.btn-icon-split(href=route('/dash/control-plane/initialize'))
|
||||
span.icon.text-white-50
|
||||
i.fas.fa-server
|
||||
span.text Initialize Master Node
|
||||
else
|
||||
p
|
||||
b Client version:
|
||||
span #{version.clientVersion.major}.#{version.clientVersion.minor} (#{version.clientVersion.gitVersion})
|
||||
span |
|
||||
b Server version:
|
||||
span #{version.serverVersion.major}.#{version.serverVersion.minor} (#{version.serverVersion.gitVersion})
|
||||
p
|
||||
a.btn.btn-warning.btn-icon-split(href=route('/dash/control-plane/upgrade'))
|
||||
span.icon.text-white-50
|
||||
i.fas.fa-upload
|
||||
span.text Upgrade Master Node
|
@ -11,6 +11,7 @@ head
|
||||
link(href='https://fonts.googleapis.com/css?family=Nunito:200,200i,300,300i,400,400i,600,600i,700,700i,800,800i,900,900i' rel='stylesheet')
|
||||
link(href=asset('dash/css/sb-admin-2.min.css') rel='stylesheet')
|
||||
link(href=asset('dash/vendor/fontawesome-free/css/all.min.css') rel='stylesheet' type='text/css')
|
||||
//link(href=asset('md5/css/mdb.min.css') rel='stylesheet' type='text/css')
|
||||
link(href=asset('dash/vendor/datatables2/dataTables.bootstrap4.min.css'))
|
||||
link(rel='stylesheet' href='https://unpkg.com/vue2-datepicker@latest/index.css')
|
||||
link(rel='stylesheet' data-name='vs/editor/editor.main' href=asset('monaco/package/min/vs/editor/editor.main.css'))
|
||||
@ -21,6 +22,10 @@ head
|
||||
| #{config('app.name')}
|
||||
hr.sidebar-divider
|
||||
.sidebar-heading Cluster
|
||||
li.nav-item
|
||||
a.nav-link(href=route('dash/control-plane'))
|
||||
i.fas.fa-fw.fa-server
|
||||
span Control Plane
|
||||
li.nav-item
|
||||
a.nav-link(href=route('dash/cobalt/listing/hostgroup'))
|
||||
i.fas.fa-fw.fa-server
|
||||
@ -111,7 +116,8 @@ a.scroll-to-top.rounded(href='#page-top')
|
||||
|
||||
block scripts
|
||||
script(src=asset('dash/vendor/jquery/jquery.min.js'))
|
||||
script(src=asset('dash/vendor/bootstrap/js/bootstrap.bundle.min.js'))
|
||||
//script(src=asset('dash/vendor/bootstrap/js/bootstrap.bundle.min.js'))
|
||||
//script(src=asset('md5/js/mdb.min.js'))
|
||||
script(src=asset('dash/vendor/jquery-easing/jquery.easing.min.js'))
|
||||
script(src=asset('vue.js'))
|
||||
script(src=asset('vues6.js') type='module')
|
||||
|
@ -11,7 +11,7 @@ export interface LXCImage {
|
||||
|
||||
@Singleton()
|
||||
export class Images {
|
||||
private imageServer = 'https://us.lxd.images.canonical.com'
|
||||
private imageServer = 'https://images.linuxcontainers.org'
|
||||
private listEndpoint = 'meta/1.0/index-system'
|
||||
private imageBinaryName = 'rootfs.tar.xz'
|
||||
|
||||
|
@ -6,7 +6,7 @@ import {
|
||||
fetch,
|
||||
Container,
|
||||
Logging,
|
||||
universalPath, uuid4,
|
||||
universalPath, uuid4, collect, ArrayElement, Collection, Maybe,
|
||||
} from '@extollo/lib'
|
||||
import { Node } from "../models/Node.model";
|
||||
import {Setting} from '../models/Setting.model'
|
||||
@ -17,6 +17,14 @@ import {unsafeESMImport} from '@extollo/lib/lib/util/unsafe'
|
||||
import * as sshpk from 'sshpk'
|
||||
import {shellCommand, SSHHost} from '../support/hosts'
|
||||
import {User} from '../models/User.model'
|
||||
import {HostGroupHost} from '../models/HostGroupHost.model'
|
||||
import {HostGroup} from '../models/HostGroup.model'
|
||||
|
||||
export interface HostUsage {
|
||||
host: HostGroupHost,
|
||||
mem: { total: number, used: number },
|
||||
cpu: { total: number, used: number },
|
||||
}
|
||||
|
||||
@Singleton()
|
||||
export class Provisioner {
|
||||
@ -31,6 +39,188 @@ export class Provisioner {
|
||||
|
||||
private cachedProxmoxApi?: Proxmox.Api
|
||||
|
||||
public async selectCandidate(group: HostGroup): Promise<Maybe<HostGroupHost>> {
|
||||
return (await this.getNodeResources(group))
|
||||
.sortByDesc(u => ((u.mem.used / u.mem.total) + (u.cpu.used / u.cpu.total)) / 2)
|
||||
.first()
|
||||
?.host
|
||||
}
|
||||
|
||||
private async getNodeResources(group: HostGroup): Promise<Collection<HostUsage>> {
|
||||
// Get the Proxmox API
|
||||
const proxmox = await this.getApi()
|
||||
|
||||
// Look up all host group hosts
|
||||
const hosts = await group.hosts().get()
|
||||
|
||||
// Query proxmox to identify the available resource on each node
|
||||
return hosts.promiseMap(async host => {
|
||||
const containers = await proxmox.nodes.$(host.pveHost.split('/')[1]).lxc.$get()
|
||||
const vms = await proxmox.nodes.$(host.pveHost.split('/')[1]).qemu.$get()
|
||||
const status = await proxmox.nodes.$(host.pveHost.split('/')[1]).status.$get()
|
||||
|
||||
const totalMem = status.memory.free
|
||||
const totalCpu = status.cpuinfo.cpus
|
||||
|
||||
const containerMem = containers.length ? collect(containers).sum(x => x.maxmem) : 0
|
||||
const vmMem = vms.length ? collect(vms).sum(x => x.maxmem) : 0
|
||||
|
||||
const containerCpu = containers.length ? collect(containers).sum(x => x.cpus) : 0
|
||||
const vmCpu = vms.length ? collect(vms).sum(x => x.cpus) : 0
|
||||
|
||||
return {
|
||||
host,
|
||||
mem: { total: totalMem, used: containerMem + vmMem },
|
||||
cpu: { total: totalCpu, used: containerCpu + vmCpu },
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private async cacheBaseImage(host: HostGroupHost): Promise<void> {
|
||||
// Get the Proxmox API!
|
||||
const proxmox = await this.getApi()
|
||||
const pveHost = proxmox.nodes.$(host!.pveHost.split('/')[1])
|
||||
const imageStorage = pveHost.storage.$('p5x.image-cache')
|
||||
|
||||
// Look up the appropriate LXC image
|
||||
this.logging.info('Searching for base LXC template image...')
|
||||
const image = await this.images.getImages()
|
||||
.then(x =>
|
||||
x.where('os', '=', 'ubuntu')
|
||||
.where('version', '=', 'jammy') // 22.04 LTS
|
||||
.where('format', '=', 'cloud')
|
||||
.where('arch', '=', 'amd64')
|
||||
.sortByDesc('timestamp')
|
||||
.first()
|
||||
)
|
||||
|
||||
if ( !image ) {
|
||||
throw new ErrorWithContext('Unable to find Ubuntu 22.04 LXC template binary')
|
||||
}
|
||||
|
||||
const imageUrl = this.images.getImageBinaryPath(image!)
|
||||
|
||||
// Upload the template image to the storage mount
|
||||
const uploadTaskUPID = await imageStorage['download-url'].$post({
|
||||
content: 'vztmpl',
|
||||
filename: 'p5x-base.tar.xz',
|
||||
url: imageUrl.toRemote,
|
||||
})
|
||||
|
||||
this.logging.info('Waiting for LXC image to upload to PVE node...')
|
||||
await this.waitForNodeTask(host!.pveHost.split('/')[1], uploadTaskUPID)
|
||||
}
|
||||
|
||||
private async getNextHostname(): Promise<string> {
|
||||
const nextNodeNum = (await Setting.load('nextNodeNum')).first() || 0
|
||||
await Setting.set('nextNodeNum', nextNodeNum + 1)
|
||||
return `p5x-node-${(Math.random() + 1).toString(36).substring(9)}${nextNodeNum}`
|
||||
}
|
||||
|
||||
public async provisionNode(group: HostGroup): Promise<Node> {
|
||||
// Look up a target node for provisioning
|
||||
const host = await this.selectCandidate(group)
|
||||
if ( !host ) {
|
||||
throw new Error('Unable to find host in host group for provisioning!')
|
||||
}
|
||||
|
||||
// Get the Proxmox API!
|
||||
const proxmox = await this.getApi()
|
||||
|
||||
// Look up the image cache on the node
|
||||
const pveHost = proxmox.nodes.$(host!.pveHost.split('/')[1])
|
||||
const imageStorage = pveHost.storage.$('p5x.image-cache')
|
||||
|
||||
// Check if the host needs to pull the image
|
||||
const imageContent = await imageStorage.content.$get()
|
||||
const hasBaseImage = imageContent.some(i =>
|
||||
i.volid === 'p5x.image-cache:vztmpl/p5x-base.tar.xz')
|
||||
|
||||
if ( !hasBaseImage ) {
|
||||
await this.cacheBaseImage(host)
|
||||
}
|
||||
|
||||
// Look up the P5x storage pool from the settings
|
||||
const pveStoragePool = await Setting.loadOneRequired('pveStoragePool')
|
||||
|
||||
// Look up the primary IP pool range
|
||||
const ipRange = await IpRange.getDefault()
|
||||
|
||||
// Get the next vmid to be used by the LXC container
|
||||
const nodeHostname = await this.getNextHostname()
|
||||
const nodeVMID = await proxmox.cluster.nextid.$get()
|
||||
const nodeIP = await ipRange.getNextAvailableOrFail()
|
||||
const sshPubKey = sshpk.parseKey(await Setting.loadOneRequired('sshPublicKey'))
|
||||
.toString('ssh')
|
||||
|
||||
// Create a new LXC container on the node for the new instance
|
||||
const nodeTaskUPID = await pveHost.lxc.$post({
|
||||
ostemplate: 'p5x.image-cache:vztmpl/p5x-base.tar.xz',
|
||||
vmid: nodeVMID,
|
||||
cores: await Setting.loadOneRequired('nodeCpus'),
|
||||
description: 'P5x Worker Node',
|
||||
hostname: nodeHostname,
|
||||
memory: await Setting.loadOneRequired('nodeRamMib'),
|
||||
net0:
|
||||
objectToKeyValue({
|
||||
name: 'eth0',
|
||||
bridge: await Setting.loadOneRequired('nodeNetworkBridge'),
|
||||
firewall: 1,
|
||||
gw: ipRange.gatewayIp,
|
||||
ip: `${nodeIP}/${ipRange.subnet}`,
|
||||
})
|
||||
.map(x => `${x.key}=${x.value}`)
|
||||
.join(','),
|
||||
onboot: true,
|
||||
password: 'strongpassword', //rootPassword, // fixme
|
||||
rootfs: `${pveStoragePool}:8`, // 8 GiB // fixme
|
||||
searchdomain: await Setting.loadOneRequired('dnsDomain'),
|
||||
'ssh-public-keys': sshPubKey,
|
||||
start: true,
|
||||
storage: pveStoragePool,
|
||||
tags: 'p5x',
|
||||
})
|
||||
|
||||
this.logging.info('Waiting for PVE node to be created...')
|
||||
await this.waitForNodeTask(host!.pveHost.split('/')[1], nodeTaskUPID)
|
||||
|
||||
// Save the node
|
||||
const node = this.container.makeNew<Node>(Node)
|
||||
node.pveId = nodeVMID
|
||||
node.hostname = nodeHostname
|
||||
node.pveHost = host!.pveHost
|
||||
node.assignedIp = nodeIP
|
||||
node.assignedSubnet = ipRange.subnet
|
||||
node.isPermanent = false
|
||||
node.isMaster = false
|
||||
await node.save()
|
||||
|
||||
// Wait for the host to come online (this will be a PCTHost passed through PVE)
|
||||
this.logging.info('Waiting for node to come online...')
|
||||
const proxyHost = await node.getHost()
|
||||
await proxyHost.waitForAlive()
|
||||
|
||||
// Configure SSH on the instance
|
||||
this.logging.info('Configuring SSH access to node...')
|
||||
await proxyHost.run(shellCommand('apt update'))
|
||||
await proxyHost.run(shellCommand('apt install -y openssh-server curl'))
|
||||
await proxyHost.run(shellCommand('systemctl enable --now ssh'))
|
||||
await new Promise<void>(res => setTimeout(() => res(), 5000))
|
||||
|
||||
// Try to re-connect to the instance directly
|
||||
this.logging.info('Attempting to re-connect to node via SSH...')
|
||||
const sshHost = await node.getHost()
|
||||
if ( !(sshHost instanceof SSHHost) ) {
|
||||
throw new ErrorWithContext('Failed to establish direct connection to node', {
|
||||
node,
|
||||
proxyHost,
|
||||
sshHost,
|
||||
})
|
||||
}
|
||||
|
||||
return node
|
||||
}
|
||||
|
||||
public async provisionMasterNode(): Promise<Node> {
|
||||
// Look up the name of the master node from the settings
|
||||
const pveMasterNode = await Setting.loadOneRequired('pveMasterNode')
|
||||
@ -79,7 +269,7 @@ export class Provisioner {
|
||||
)
|
||||
|
||||
if ( !image ) {
|
||||
throw new ErrorWithContext('Unable to find Debian Bookworm LXC template binary')
|
||||
throw new ErrorWithContext('Unable to find Ubuntu 22.04 LXC template binary')
|
||||
}
|
||||
|
||||
const imageUrl = this.images.getImageBinaryPath(image!)
|
||||
@ -136,6 +326,7 @@ export class Provisioner {
|
||||
const node = this.container.makeNew<Node>(Node)
|
||||
node.pveId = masterVMID
|
||||
node.hostname = 'p5x-control'
|
||||
node.pveHost = `node/${pveMasterNode}`
|
||||
node.assignedIp = masterIP
|
||||
node.assignedSubnet = ipRange.subnet
|
||||
node.isPermanent = true
|
||||
|
@ -1,8 +1,8 @@
|
||||
import {
|
||||
Awaitable,
|
||||
AwareOfContainerLifecycle, ErrorWithContext,
|
||||
Filesystem,
|
||||
infer,
|
||||
Filesystem, hasOwnProperty,
|
||||
infer, Maybe,
|
||||
objectToKeyValue,
|
||||
UniversalPath,
|
||||
uuid4,
|
||||
@ -12,6 +12,50 @@ import {ExecutionResult} from './ExecutionResult'
|
||||
import {CommandError} from './errors'
|
||||
import {SystemMetrics} from './SystemMetrics'
|
||||
|
||||
export interface SystemDUnitDetails {
|
||||
Type: string
|
||||
Result: 'success'|string
|
||||
NRestarts: number
|
||||
Id: string
|
||||
Names: string[]
|
||||
Requires?: string[]
|
||||
Wants?: string[]
|
||||
WantedBy?: string
|
||||
Description?: string
|
||||
ActiveState?: 'active'|string
|
||||
}
|
||||
|
||||
export const isSystemDUnitDetails = (what: unknown): what is SystemDUnitDetails => {
|
||||
return !!(
|
||||
typeof what === 'object'
|
||||
&& what
|
||||
&& hasOwnProperty(what, 'Type') && typeof what.Type === 'string'
|
||||
&& hasOwnProperty(what, 'Result') && typeof what.Result === 'string'
|
||||
&& hasOwnProperty(what, 'NRestarts') && typeof what.NRestarts === 'number'
|
||||
&& hasOwnProperty(what, 'Id') && typeof what.Id === 'string'
|
||||
&& (
|
||||
(hasOwnProperty(what, 'Requires') && Array.isArray(what.Requires) && what.Requires.every(x => typeof x === 'string'))
|
||||
|| !hasOwnProperty(what, 'Requires')
|
||||
)
|
||||
&& (
|
||||
(hasOwnProperty(what, 'Wants') && Array.isArray(what.Wants) && what.Wants.every(x => typeof x === 'string'))
|
||||
|| !hasOwnProperty(what, 'Wants')
|
||||
)
|
||||
&& (
|
||||
(hasOwnProperty(what, 'WantedBy') && typeof what.WantedBy === 'string')
|
||||
|| !hasOwnProperty(what, 'WantedBy')
|
||||
)
|
||||
&& (
|
||||
(hasOwnProperty(what, 'Description') && typeof what.Description === 'string')
|
||||
|| !hasOwnProperty(what, 'Description')
|
||||
)
|
||||
&& (
|
||||
(hasOwnProperty(what, 'ActiveState') && typeof what.ActiveState === 'string')
|
||||
|| !hasOwnProperty(what, 'ActiveState')
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
export class StandardCommandPalette implements CommandPalette {
|
||||
changeDirectory = shellCommand(`cd "%%PATH%%"`)
|
||||
cpuPercentage = shellCommand(`grep 'cpu ' /proc/stat | awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage}'`)
|
||||
@ -30,6 +74,7 @@ export class StandardCommandPalette implements CommandPalette {
|
||||
tempFile = shellCommand(`mktemp`)
|
||||
tempPath = shellCommand(`mktemp -d`)
|
||||
echo = shellCommand(`echo "%%OUTPUT%%"`)
|
||||
systemDUnitShow = shellCommand(`systemctl show --no-pager %%UNIT%%`)
|
||||
|
||||
public format(command: keyof CommandPaletteCommands, substitutions: Record<string, any>): ShellCommand {
|
||||
let formatted = `${this[command]}`
|
||||
@ -200,4 +245,34 @@ export abstract class Host implements AwareOfContainerLifecycle {
|
||||
public reboot(): Awaitable<ExecutionResult> {
|
||||
return this.run(this.getCommandPalette().format('reboot', {}))
|
||||
}
|
||||
|
||||
public async getSystemDUnit(unit: string): Promise<Maybe<SystemDUnitDetails>> {
|
||||
const cmd = this.getCommandPalette().format('systemDUnitShow', { unit })
|
||||
const result = await this.run(cmd)
|
||||
const listing = result.standardOutput.join('\n')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(x => ({
|
||||
key: x.split('=')[0],
|
||||
value: x.split('=').slice(1).join('='),
|
||||
}))
|
||||
.map((x: {key: string, value: any}) => {
|
||||
if ( ['Names', 'Requires', 'Wants'].includes(x.key) ) {
|
||||
x.value = x.value.split(' ').filter(Boolean)
|
||||
} else {
|
||||
x.value = infer(x.value)
|
||||
}
|
||||
|
||||
return {[x.key]: x.value}
|
||||
})
|
||||
.reduce((a, b) => ({...a, ...b}))
|
||||
|
||||
if ( hasOwnProperty(listing, 'LoadError') ) {
|
||||
return
|
||||
}
|
||||
|
||||
if ( isSystemDUnitDetails(listing) ) {
|
||||
return listing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -22,6 +22,7 @@ export interface CommandPaletteCommands {
|
||||
fileDirectoryOwnershipSetFlat: ShellCommand
|
||||
fileDirectoryOwnershipSetRecursive: ShellCommand
|
||||
echo: ShellCommand
|
||||
systemDUnitShow: ShellCommand
|
||||
}
|
||||
|
||||
export type CommandPalette = CommandPaletteCommands & {
|
||||
|
88
src/app/support/k8s/ControlPlane.ts
Normal file
88
src/app/support/k8s/ControlPlane.ts
Normal file
@ -0,0 +1,88 @@
|
||||
import {Host, ShellCommand, shellCommand} from '../hosts'
|
||||
import {hasOwnProperty} from '@extollo/lib'
|
||||
import {Node} from '../../models/Node.model'
|
||||
|
||||
|
||||
interface ControlPlaneSideVersion {
|
||||
major: string,
|
||||
minor: string,
|
||||
gitVersion: string,
|
||||
goVersion: string,
|
||||
}
|
||||
|
||||
const isControlPlaneSideVersion = (what: unknown): what is ControlPlaneSideVersion => {
|
||||
return !!(
|
||||
typeof what === 'object'
|
||||
&& what
|
||||
&& hasOwnProperty(what, 'major') && typeof what.major === 'string'
|
||||
&& hasOwnProperty(what, 'minor') && typeof what.minor === 'string'
|
||||
&& hasOwnProperty(what, 'gitVersion') && typeof what.gitVersion === 'string'
|
||||
&& hasOwnProperty(what, 'goVersion') && typeof what.goVersion === 'string'
|
||||
)
|
||||
}
|
||||
|
||||
export interface ControlPlaneVersion {
|
||||
clientVersion: ControlPlaneSideVersion,
|
||||
serverVersion: ControlPlaneSideVersion,
|
||||
}
|
||||
|
||||
export const isControlPlaneVersion = (what: unknown): what is ControlPlaneVersion => {
|
||||
return !!(
|
||||
typeof what === 'object'
|
||||
&& what
|
||||
&& hasOwnProperty(what, 'clientVersion') && isControlPlaneSideVersion(what.clientVersion)
|
||||
&& hasOwnProperty(what, 'serverVersion') && isControlPlaneSideVersion(what.serverVersion)
|
||||
)
|
||||
}
|
||||
|
||||
export class ControlPlane {
|
||||
constructor(
|
||||
private host: Host
|
||||
) {}
|
||||
|
||||
public async hasKubelet(): Promise<boolean> {
|
||||
return !!(await this.host.getSystemDUnit('k3s'))
|
||||
}
|
||||
|
||||
public async getVersion(node?: Node): Promise<ControlPlaneVersion> {
|
||||
const host = node ? (await node.getHost()) : this.host
|
||||
const result = await host.run(shellCommand('kubectl version --output=json'))
|
||||
const version = JSON.parse(result.standardOutput.join('\n'))
|
||||
if ( !isControlPlaneVersion(version) ) {
|
||||
throw new Error('Invalid response from control plane: ' + result.standardOutput.join('\n'))
|
||||
}
|
||||
|
||||
return version
|
||||
}
|
||||
|
||||
public async initializeMaster(): Promise<ControlPlaneVersion> {
|
||||
await this.host.run(shellCommand('apt-get update'))
|
||||
await this.host.run(shellCommand('apt-get install -y curl'))
|
||||
await this.host.run(await this.getK3sMasterCommand())
|
||||
return this.getVersion()
|
||||
}
|
||||
|
||||
public async initializeNode(node: Node): Promise<ControlPlaneVersion> {
|
||||
const host = await node.getHost()
|
||||
await host.run(shellCommand('apt-get update'))
|
||||
await host.run(shellCommand('apt-get install -y curl'))
|
||||
await host.run(await this.getK3sNodeCommand(node))
|
||||
return this.getVersion(node)
|
||||
}
|
||||
|
||||
private async getK3sNodeCommand(node: Node): Promise<ShellCommand> {
|
||||
const master = await Node.getMaster()
|
||||
const token = await this.getMasterToken()
|
||||
return shellCommand(`curl -fsL https://get.k3s.io | K3S_URL=https://${master.assignedIp}:6443 K3S_TOKEN=${token} sh -s - --node-name ${node.hostname}`)
|
||||
}
|
||||
|
||||
private async getMasterToken(): Promise<string> {
|
||||
const result = await this.host.run(shellCommand('cat /var/lib/rancher/k3s/server/node-token'))
|
||||
return result.standardOutput.join('\n').trim()
|
||||
}
|
||||
|
||||
private async getK3sMasterCommand(): Promise<ShellCommand> {
|
||||
const master = await Node.getMaster()
|
||||
return shellCommand(`curl -fsL https://get.k3s.io | sh -s - --disable traefik --node-name ${master.hostname}`)
|
||||
}
|
||||
}
|
@ -28,4 +28,5 @@ export type Settings = Record<string, JSONState> & {
|
||||
nodeNetworkBridge: string
|
||||
nodeCpus: number
|
||||
nodeRamMib: number
|
||||
nextNodeNum: number
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user