p5x/api-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Start work on remote data sources; add infrastructure for host groups

Browse Source
This commit is contained in:
Garrett Mills 2023-10-09 23:40:14 -05:00
parent a7d273da1c
commit 17fda7b6ef
15 changed files with 549 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/app/cobalt.ts
View File

@ -1,4 +1,4 @@
import {OrderDirection, QueryRow} from '@extollo/lib'
import {Awaitable, Constructable, Maybe, OrderDirection, QueryRow} from '@extollo/lib'
export enum FieldType {
text = 'text',
@ -34,17 +34,27 @@ const allResourceActions = [
ResourceAction.update, ResourceAction.delete,
]
export type CobaltAction = {
slug: string,
title: string,
color: string,
icon: string,
type: 'route',
route: string,
overall?: boolean,
}
export type FieldBase = {
key: string,
display: string,
type: FieldType,
required: boolean,
queryable?: boolean,
sort?: 'asc' | 'desc',
renderer?: Renderer,
readonly?: boolean,
helpText?: string,
placeholder?: string,
queryable?: boolean,
hideOn?: {
form?: boolean,
listing?: boolean,
@ -53,25 +63,45 @@ export type FieldBase = {
export type SelectOptions = {display: string, value: any}[]
export type RemoteSelectDefinition = {
displayFrom: string,
valueFrom: string,
source: DataSource,
}
export type FieldDefinition = FieldBase
| FieldBase & { type: FieldType.select, options: SelectOptions }
| FieldBase & { type: FieldType.select } & RemoteSelectDefinition
export interface DataSourceController {
read(): Awaitable<QueryRow[]>
readOne(id: any): Awaitable<Maybe<QueryRow>>
insert(row: QueryRow): Awaitable<QueryRow>
update(id: any, row: QueryRow): Awaitable<QueryRow>
delete(id: any): Awaitable<void>
}
export type DataSource =
{ collection: string }
| { controller: Constructable<DataSourceController> }
| { method: Constructable<() => Awaitable<QueryRow[]>> }
export interface ResourceConfiguration {
key: string,
collection: string,
source: DataSource,
primaryKey: string,
orderField?: string,
orderDirection?: OrderDirection,
generateKeyOnInsert?: () => string|number,
process?: {
afterRead?: (row: QueryRow) => QueryRow,
},
processBeforeInsert?: (row: QueryRow) => Awaitable<QueryRow>,
processAfterRead?: (row: QueryRow) => Awaitable<QueryRow>,
display: {
field?: string,
singular: string,
plural: string,
},
supportedActions: ResourceAction[],
otherActions: CobaltAction[],
fields: FieldDefinition[],
}

62
src/app/configs/cobalt.config.ts
View File

@ -1,4 +1,6 @@
import {FieldType, Renderer, ResourceAction, ResourceConfiguration} from '../cobalt'
import {allResourceActions, FieldType, Renderer, ResourceAction, ResourceConfiguration} from '../cobalt'
import {constructable, QueryRow} from '@extollo/lib'
import {HostGroups} from '../http/controllers/resource/HostGroups.controller'
const tapRef = <T>(op: ((t: T) => unknown)): ((opd: T) => T) => {
return (opd: T): T => {
@ -9,10 +11,55 @@ const tapRef = <T>(op: ((t: T) => unknown)): ((opd: T) => T) => {
export default {
resources: [
{
key: 'hostgroup',
primaryKey: 'id',
source: { controller: constructable(HostGroups) },
display: {
field: 'name',
singular: 'Host Group',
plural: 'Host Groups',
},
orderField: 'name',
orderDirection: 'asc',
supportedActions: allResourceActions,
fields: [
{
key: 'id',
display: 'Host Group ID',
type: FieldType.number,
readonly: true,
required: false,
},
{
key: 'name',
display: 'Name',
type: FieldType.text,
required: true,
},
{
key: 'hosts',
display: 'Physical Hosts',
type: FieldType.select,
required: true,
displayFrom: 'pveDisplay',
valueFrom: 'pveHost',
hideOn: {
listing: true,
},
source: {
method: constructable<HostGroups>(HostGroups)
.tap(c => c.getBoundMethod('getPVEHosts')),
},
},
],
},
{
key: 'node',
primaryKey: 'id',
source: {
collection: 'p5x_nodes',
},
display: {
field: 'hostname',
singular: 'Instance',
@ -21,9 +68,18 @@ export default {
orderField: 'pve_id',
orderDirection: 'asc',
supportedActions: [ResourceAction.read, ResourceAction.readOne],
process: {
afterRead: tapRef(qr => qr.ip_display = `<pre><code>${qr.assigned_ip}/${qr.assigned_subnet}</code></pre>`),
otherActions: [
{
slug: 'provision',
title: 'Provision',
color: 'success',
icon: 'fa-plus',
type: 'route',
route: '/dash/provision',
overall: true,
},
],
processAfterRead: tapRef((qr: QueryRow) => qr.ip_display = `<pre><code>${qr.assigned_ip}/${qr.assigned_subnet}</code></pre>`),
fields: [
{
key: 'id',

6
src/app/http/controllers/Dash.controller.ts
View File

@ -13,4 +13,10 @@ export class Dash extends Controller {
public main(): ResponseObject {
return view('dash:template')
}
public provision(): ResponseObject {
return view('dash:provision', {
})
}
}

214
src/app/http/controllers/cobalt/ResourceAPI.controller.ts
View File

@ -1,7 +1,7 @@
import {
api,
api, ArrayIterable, AsyncCollection, AsyncPipe, Awaitable,
Builder,
collect,
collect, Collection,
Config,
Controller,
DatabaseService,
@ -10,14 +10,64 @@ import {
HTTPError,
HTTPStatus,
Inject,
Injectable,
Injectable, Iterable,
Maybe,
QueryRow,
} from '@extollo/lib'
import {FieldDefinition, FieldType, ResourceAction, ResourceConfiguration, SelectOptions} from '../../../cobalt'
import {
DataSource,
FieldDefinition,
FieldType,
ResourceAction,
ResourceConfiguration,
SelectOptions,
} from '../../../cobalt'
const parser = require('any-date-parser')
class AwaitableIterable<T> extends Iterable<T> {
public static lift<TInner>(source: Awaitable<Iterable<TInner>>): AwaitableIterable<TInner> {
return new AwaitableIterable<TInner>(source)
}
private resolved?: Iterable<T>
constructor(
protected source: Awaitable<Iterable<T>>
) {
super()
}
at(i: number): Promise<Maybe<T>> {
return this.resolve()
.then(x => x.at(i))
}
clone(): Iterable<T> {
return AwaitableIterable.lift(
this.resolve()
.then(x => x.clone())
)
}
range(start: number, end: number): Promise<Collection<T>> {
return this.resolve()
.then(x => x.range(start, end))
}
count(): Promise<number> {
return this.resolve()
.then(x => x.count())
}
private async resolve(): Promise<Iterable<T>> {
if ( !this.resolved ) {
this.resolved = await this.source
}
return this.resolved
}
}
@Injectable()
export class ResourceAPI extends Controller {
@Inject()
@ -26,8 +76,8 @@ export class ResourceAPI extends Controller {
@Inject()
protected readonly db!: DatabaseService
public configure(key: string) {
const config = this.getResourceConfig(key)
public async configure(key: string) {
const config = await this.getResourceConfig(key)
if ( config ) {
return api.one(config)
}
@ -36,54 +86,73 @@ export class ResourceAPI extends Controller {
}
public async read(key: string) {
const config = this.getResourceConfigOrFail(key)
const config = await this.getResourceConfigOrFail(key)
this.checkAction(config, ResourceAction.read)
let result = await this.make<Builder>(Builder)
let result: QueryRow[]
if ( hasOwnProperty(config.source, 'controller') ) {
const controller = config.source.controller.apply(this.request)
result = await controller.read()
} else if ( hasOwnProperty(config.source, 'method') ) {
const method = config.source.method.apply(this.request)
result = await method()
} else {
result = await this.make<Builder>(Builder)
.select(config.primaryKey, ...config.fields.filter(x => x.queryable !== false).map(x => x.key))
.from(config.collection)
.from(config.source.collection)
.limit(500)
.orderBy(config.orderField || config.primaryKey, config.orderDirection || 'asc')
.connection(this.db.get())
.get()
.all()
}
if ( config.process?.afterRead ) {
result = result.map(config.process.afterRead)
if ( config.processAfterRead ) {
result = await Promise.all(result.map(config.processAfterRead))
}
return api.many(result)
}
public async readOne(key: string, id: number|string) {
const config = this.getResourceConfigOrFail(key)
const config = await this.getResourceConfigOrFail(key)
this.checkAction(config, ResourceAction.readOne)
let row = await this.make<Builder>(Builder)
let row: Maybe<QueryRow>
if ( hasOwnProperty(config.source, 'controller') ) {
const controller = config.source.controller.apply(this.request)
row = await controller.readOne(id)
} else if ( hasOwnProperty(config.source, 'method') ) {
const method = config.source.method.apply(this.request)
row = collect(await method())
.firstWhere(config.primaryKey, '=', key)
} else {
row = await this.make<Builder>(Builder)
.select(config.primaryKey, ...config.fields.filter(x => x.queryable !== false).map(x => x.key))
.from(config.collection)
.from(config.source.collection)
.where(config.primaryKey, '=', id)
.limit(1)
.connection(this.db.get())
.first()
}
if ( !row ) {
throw new HTTPError(HTTPStatus.NOT_FOUND)
}
if ( config.process?.afterRead ) {
row = config.process.afterRead(row)
if ( config.processAfterRead ) {
row = await config.processAfterRead(row)
}
return api.one(row)
}
public async create(key: string, dataContainer: DataContainer) {
const config = this.getResourceConfigOrFail(key)
const config = await this.getResourceConfigOrFail(key)
this.checkAction(config, ResourceAction.create)
// Load input values
const queryRow: QueryRow = {}
let queryRow: QueryRow = {}
for ( const field of config.fields ) {
const value = dataContainer.input(field.key)
if ( field.required && typeof value === 'undefined' ) {
@ -97,20 +166,32 @@ export class ResourceAPI extends Controller {
queryRow[config.primaryKey] = config.generateKeyOnInsert()
}
if ( config.processBeforeInsert ) {
queryRow = await config.processBeforeInsert(queryRow)
}
// Create insert query
const result = await this.make<Builder>(Builder)
.table(config.collection)
let result: Maybe<QueryRow>
if ( hasOwnProperty(config.source, 'controller') ) {
const controller = config.source.controller.apply(this.request)
result = await controller.insert(queryRow)
} else if ( hasOwnProperty(config.source, 'method') ) {
throw new Error('The "method" source type does not support creating records.')
} else {
result = await this.make<Builder>(Builder)
.table(config.source.collection)
.returning(config.primaryKey, ...config.fields.map(x => x.key))
.connection(this.db.get())
.insert(queryRow)
.then(x => x.rows.first())
}
// Return result
return api.one(result)
}
public async update(key: string, id: number|string, dataContainer: DataContainer) {
const config = this.getResourceConfigOrFail(key)
const config = await this.getResourceConfigOrFail(key)
this.checkAction(config, ResourceAction.update)
// Load input values
@ -128,31 +209,46 @@ export class ResourceAPI extends Controller {
await this.readOne(key, id)
// Create update query
const result = await this.make<Builder>(Builder)
.table(config.collection)
let result: Maybe<QueryRow>
if ( hasOwnProperty(config.source, 'controller') ) {
const controller = config.source.controller.apply(this.request)
result = await controller.update(id, queryRow)
} else if ( hasOwnProperty(config.source, 'method') ) {
throw new Error('The "method" source type does not support updating records.')
} else {
result = await this.make<Builder>(Builder)
.table(config.source.collection)
.returning(config.primaryKey, ...config.fields.map(x => x.key))
.connection(this.db.get())
.where(config.primaryKey, '=', id)
.update(queryRow)
.then(x => x.rows.first())
}
// Return the result
return api.one(result)
}
public async delete(key: string, id: number|string) {
const config = this.getResourceConfigOrFail(key)
const config = await this.getResourceConfigOrFail(key)
this.checkAction(config, ResourceAction.delete)
// Make sure the row exists
await this.readOne(key, id)
// Execute the query
if ( hasOwnProperty(config.source, 'controller') ) {
const controller = config.source.controller.apply(this.request)
await controller.delete(id)
} else if ( hasOwnProperty(config.source, 'method') ) {
throw new Error('The "method" source type does not support deleting records.')
} else {
await this.make<Builder>(Builder)
.table(config.collection)
.table(config.source.collection)
.connection(this.db.get())
.where(config.primaryKey, '=', id)
.delete()
}
return { success: true }
}
@ -224,20 +320,76 @@ export class ResourceAPI extends Controller {
}
}
protected getResourceConfigOrFail(key: string): ResourceConfiguration {
const config = this.getResourceConfig(key)
protected async getResourceConfigOrFail(key: string): Promise<ResourceConfiguration> {
const config = await this.getResourceConfig(key)
if ( !config ) {
throw new HTTPError(HTTPStatus.NOT_FOUND)
}
return config
}
protected getResourceConfig(key: string): Maybe<ResourceConfiguration> {
protected async getResourceConfig(key: string): Promise<Maybe<ResourceConfiguration>> {
const configs = this.config.get('cobalt.resources') as ResourceConfiguration[]
for ( const config of configs ) {
if ( config.key === key ) {
let config: Maybe<ResourceConfiguration> = undefined
for ( const match of configs ) {
if ( match.key === key ) {
config = match
break
}
}
if ( !config ) {
return config
}
}
// Resolve `select` fields with remote sources
config.fields = await collect(config.fields)
.map(async field => {
if ( field.type === FieldType.select && hasOwnProperty(field, 'source') ) {
return {
...field,
options: await this.readSource(field.source)
.map(qr => ({
display: qr[field.displayFrom],
value: qr[field.valueFrom],
}))
.then(c => c.all())
} as FieldDefinition
}
return field
})
.awaitAll()
.then(x => x.all())
return config
}
protected readSource(source: DataSource): AsyncCollection<QueryRow> {
let iter: Iterable<QueryRow>
if ( hasOwnProperty(source, 'controller') ) {
iter = AwaitableIterable.lift(
AsyncPipe.wrap(source.controller.apply(this.request))
.tap(controller => controller.read())
.tap(rows => new ArrayIterable(rows))
.resolve()
)
} else if ( hasOwnProperty(source, 'method') ) {
iter = AwaitableIterable.lift(
AsyncPipe.wrap(source.method.apply(this.request))
.tap(method => method())
.tap(rows => new ArrayIterable(rows))
.resolve()
)
} else {
iter = this.make<Builder>(Builder)
.from(source.collection)
.limit(500)
.connection(this.db.get())
.getResultIterable()
}
return new AsyncCollection(iter)
}
}

80
src/app/http/controllers/resource/HostGroups.controller.ts Normal file
View File

@ -0,0 +1,80 @@
import {AsyncPipe, Awaitable, Controller, Inject, Injectable, Maybe, QueryRow, Safe} from '@extollo/lib'
import {DataSourceController} from '../../../cobalt'
import {HostGroup} from '../../../models/HostGroup.model'
import {Provisioner} from '../../../services/Provisioner.service'
@Injectable()
export class HostGroups extends Controller implements DataSourceController {
@Inject()
protected readonly provisioner!: Provisioner
read(): Awaitable<QueryRow[]> {
return HostGroup.query<HostGroup>()
.orderBy('name')
.get()
.collect()
.then(x => x.mapCall('toCobalt').awaitAll())
.then(x => x.all())
}
readOne(id: any): Awaitable<Maybe<QueryRow>> {
return HostGroup.query<HostGroup>()
.whereKey(id)
.first()
.then(x => x?.toCobalt())
}
insert(row: QueryRow): Awaitable<QueryRow> {
const hosts = row.hosts
if ( !Array.isArray(hosts) || !hosts.every(i => typeof i === 'string') ) {
throw new Error('Invalid hosts: must be number[]')
}
return AsyncPipe.wrap(this.request.makeNew<HostGroup>(HostGroup))
.peek(g => g.name = (new Safe(row.name)).string())
.peek(g => g.save())
.peek(g => g.setHosts(hosts))
.tap(g => g.toCobalt())
.resolve()
}
async update(id: any, row: QueryRow): Promise<QueryRow> {
const hosts = row.hosts
if ( !Array.isArray(hosts) || !hosts.every(i => typeof i === 'string') ) {
throw new Error('Invalid hosts: must be number[]')
}
const hostGroup = await HostGroup.query<HostGroup>()
.whereKey(id)
.first()
if ( !hostGroup ) {
throw new Error('Invalid host group ID.')
}
hostGroup.name = (new Safe(row.name)).string()
await hostGroup.save()
await hostGroup.setHosts(hosts)
return hostGroup.toCobalt()
}
async delete(id: any): Promise<void> {
HostGroup.query<HostGroup>()
.whereKey(id)
.delete()
}
async getPVEHosts(): Promise<QueryRow[]> {
return AsyncPipe.wrap(this.provisioner)
.tap(p => p.getApi())
.tap(api => api.nodes.$get())
.tap(nodes =>
nodes.map(node => ({
pveHost: node.id || node.node,
pveDisplay: `${node.node} (CPUs: ${node.maxcpu}, RAM: ${Math.floor((node.maxmem || 0) / 1000000000)})`
}))
)
.resolve()
}
}

3
src/app/http/routes/dash.routes.ts
View File

@ -30,6 +30,9 @@ Route
.alias('@dash')
.calls<Dash>(Dash, dash => dash.main)
Route.get('/provision')
.calls<Dash>(Dash, dash => dash.provision)
Route.group('/cobalt/resource', () => {
Route.get('/:key/configure')
.parameterMiddleware(parseKey)

46
src/app/migrations/2023-10-10T03:04:47.462Z_CreatePhysicalHostsTableMigration.migration.ts Normal file
View File

@ -0,0 +1,46 @@
import {Injectable, Migration, Inject, DatabaseService, FieldType} from '@extollo/lib'
/**
* CreatePhysicalHostsTableMigration
* ----------------------------------
* Put some description here.
*/
@Injectable()
export default class CreatePhysicalHostsTableMigration extends Migration {
@Inject()
protected readonly db!: DatabaseService
/**
* Apply the migration.
*/
async up(): Promise<void> {
const schema = this.db.get().schema()
const hostGroups = await schema.table('p5x_host_groups')
hostGroups.primaryKey('id')
hostGroups.column('name').type(FieldType.varchar)
await schema.commit(hostGroups)
const hostGroupHosts = await schema.table('p5x_host_group_hosts')
hostGroupHosts.primaryKey('id')
hostGroupHosts.column('pve_host').type(FieldType.varchar)
hostGroupHosts.column('host_group_id').type(FieldType.int)
hostGroupHosts.index('hostgrouphostid').field('host_group_id')
await schema.commit(hostGroupHosts)
}
/**
* Undo the migration.
*/
async down(): Promise<void> {
const schema = this.db.get().schema()
const hostGroups = await schema.table('p5x_host_groups')
hostGroups.dropIfExists()
await schema.commit(hostGroups)
const hostGroupHosts = await schema.table('p5x_host_group_hosts')
hostGroupHosts.dropIfExists()
await schema.commit(hostGroupHosts)
}
}

48
src/app/models/HostGroup.model.ts Normal file
View File

@ -0,0 +1,48 @@
import {Injectable, Model, Field, FieldType, collect, Related, QueryRow} from '@extollo/lib'
import {HostGroupHost} from './HostGroupHost.model'
/**
* HostGroup Model
* -----------------------------------
* Put some description here.
*/
@Injectable()
export class HostGroup extends Model<HostGroup> {
protected static table = 'p5x_host_groups'
protected static key = 'id'
@Field(FieldType.serial)
public id?: number
@Field(FieldType.varchar)
public name!: string
@Related()
public hosts() {
return this.hasMany(HostGroupHost, 'id', 'hostGroupId')
}
public async setHosts(pveHosts: string[]): Promise<void> {
if ( !this.id ) {
throw new Error('Cannot setHosts on HostGroup that has not yet been persisted')
}
// Remove any existing hosts
await this.hosts().query().delete()
// Insert the host records
const rows = pveHosts.map(pveHost => ({
pveHost,
hostGroupId: this.key(),
}))
await HostGroupHost.query<HostGroupHost>()
.insert(rows)
}
public async toCobalt(): Promise<QueryRow> {
const obj = this.toQueryRow()
obj.hosts = await this.hosts().get().then(x => x.pluck('pveHost'))
return obj
}
}

21
src/app/models/HostGroupHost.model.ts Normal file
View File

@ -0,0 +1,21 @@
import {Field, FieldType, Injectable, Model} from '@extollo/lib'
/**
* HostGroupHost Model
* -----------------------------------
* Put some description here.
*/
@Injectable()
export class HostGroupHost extends Model<HostGroupHost> {
protected static table = 'p5x_host_group_hosts'
protected static key = 'id'
@Field(FieldType.serial)
public id?: number
@Field(FieldType.varchar, 'pve_host')
public pveHost!: string
@Field(FieldType.int, 'host_group_id')
public hostGroupId!: number
}

5
src/app/resources/assets/cobalt/Resource.js
View File

@ -32,6 +32,7 @@ export class Resource {
this.key = key
this.configuration = {
supportedActions: [],
otherActions: [],
}
}
@ -51,6 +52,10 @@ export class Resource {
return this.configuration.supportedActions
}
getOtherActions() {
return this.configuration.otherActions || []
}
supports(action) {
return this.getSupportedActions().includes(action)
}

1
src/app/resources/assets/cobalt/component/Form.component.js
View File

@ -21,7 +21,6 @@ const template = `
:id="id + field.key"
:aria-describedby="id + field.key + '_help'"
:readonly="mode === 'view' || field.readonly"
:onclick="(mode === 'view' || field.readonly) ? 'return false;' : ''"
:required="field.required"
>
<label for="id + field.key">{{ field.display }}</label>

9
src/app/resources/assets/cobalt/component/Listing.component.js
View File

@ -119,7 +119,7 @@ export class ListingComponent extends Component {
action: 'update',
defer: true,
})
} else if ( !reload && this.resource.supports(ResourceActions.read) ) {
} else if ( !reload && this.resource.supports(ResourceActions.readOne) ) {
this.actions.push({
title: 'View',
color: 'primary',
@ -143,6 +143,13 @@ export class ListingComponent extends Component {
})
}
for ( const action of this.resource.getOtherActions() ) {
this.actions.push({
resource: this.resource.key,
...action,
})
}
this.rows = await this.resource.read()
this.rows.forEach((row, idx) => row.idx = idx)

20
src/app/resources/assets/cobalt/service/Action.service.js
View File

@ -14,6 +14,7 @@ export class ActionService {
async perform(action, data, onComplete = () => {}) {
if ( action.type === 'back' ) this.goBack()
if ( action.type === 'resource' ) await this.handleResourceAction(action, data, onComplete)
if ( action.type === 'route' ) await this.handleRouteAction(action, data, onComplete)
}
async handleResourceAction(action, data, onComplete) {
@ -60,6 +61,25 @@ export class ActionService {
location.assign(Session.get().url(`dash/cobalt/form/${key}${id ? '/' + id : ''}${view ? '/view' : ''}`))
}
async handleRouteAction(action, data, onComplete = () => {}) {
let route = Session.get().url(action.route)
route = this.replacePlaceholders(route, { resourceKey: action.resource })
route = this.replacePlaceholders(route, data)
location.assign(route)
}
replacePlaceholders(str, data) {
for ( const key in data ) {
if ( !data.hasOwnProperty(key) ) {
continue
}
str = String(str).replaceAll(':' + key, data[key])
}
return str
}
goBack() {
history.back()
}

4
src/app/resources/views/dash/provision.pug Normal file
View File

@ -0,0 +1,4 @@
extends template
block content
h1 Provision!

4
src/app/resources/views/dash/template.pug
View File

@ -21,6 +21,10 @@ head
| #{config('app.name')}
hr.sidebar-divider
.sidebar-heading Cluster
li.nav-item
a.nav-link(href=route('dash/cobalt/listing/hostgroup'))
i.fas.fa-fw.fa-server
span Host Groups
li.nav-item
a.nav-link(href=route('dash/cobalt/listing/node'))
i.fas.fa-fw.fa-server