From ed7b1671677ba093697e21791907e1e6e08bf2a1 Mon Sep 17 00:00:00 2001 From: garrettmills Date: Tue, 25 Feb 2025 00:18:45 -0500 Subject: [PATCH] Start roughing out K8s deployment spec for api-server; bootstrap global locks AFTER DB migrations have run --- Dockerfile | 11 ++------ deploy/.gitignore | 1 + deploy/0-namespace.yaml | 5 ++++ deploy/20-volume.yaml | 27 ++++++++++++++++++++ deploy/22-volumeclaim.yaml | 14 +++++++++++ deploy/40-statefulset.yaml | 51 ++++++++++++++++++++++++++++++++++++++ scripts/build.sh | 2 +- scripts/push.sh | 3 +++ scripts/run.sh | 2 +- src/api/db/mod.rs | 3 --- src/api/mod.rs | 15 ++++++++++- 11 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 deploy/.gitignore create mode 100644 deploy/0-namespace.yaml create mode 100644 deploy/20-volume.yaml create mode 100644 deploy/22-volumeclaim.yaml create mode 100644 deploy/40-statefulset.yaml create mode 100755 scripts/push.sh diff --git a/Dockerfile b/Dockerfile index c2b5348..626bafb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,14 @@ FROM ubuntu:24.04 -RUN groupadd --gid 3450 p5x \ - && useradd --uid 3450 --gid 3450 p5x \ - && mkdir -p /p5x/data \ - && chown -R p5x:p5x /p5x \ - && chmod 755 /p5x \ - && usermod -d /p5x p5x +RUN mkdir -p /p5x/data COPY ./target/x86_64-unknown-linux-gnu/release/p5x /p5x/p5x COPY Rocket.toml /p5x/Rocket.toml -WORKDIR /p5x -USER p5x:p5x - ENV ROCKET_PROFILE="docker" ENV RUST_LOG="p5x=debug,sqlx=warn,info" ENV P5X_SSH_PUBKEY_PATH="/p5x/data/ssh-pubkey.pem" ENV P5X_SSH_PRIVKEY_PATH="/p5x/data/ssh-privkey.pem" +WORKDIR /p5x CMD ["/p5x/p5x", "api-server"] diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 0000000..8faf276 --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1 @@ +10-environment.yaml diff --git a/deploy/0-namespace.yaml b/deploy/0-namespace.yaml new file mode 100644 index 0000000..7070042 --- /dev/null +++ b/deploy/0-namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: p5x-system diff --git a/deploy/20-volume.yaml b/deploy/20-volume.yaml new file mode 100644 index 0000000..4729446 --- /dev/null +++ b/deploy/20-volume.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: system-data + namespace: p5x-system +spec: + capacity: + storage: 5Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: local-storage + local: + path: /mnt/p5x-system-data + claimRef: + namespace: p5x-system + name: system-data + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - control.k8s diff --git a/deploy/22-volumeclaim.yaml b/deploy/22-volumeclaim.yaml new file mode 100644 index 0000000..36db2ce --- /dev/null +++ b/deploy/22-volumeclaim.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: system-data + namespace: p5x-system +spec: + storageClassName: local + accessModes: + - ReadWriteOnce + volumeName: system-data + resources: + requests: + storage: 5Gi diff --git a/deploy/40-statefulset.yaml b/deploy/40-statefulset.yaml new file mode 100644 index 0000000..48e5a1c --- /dev/null +++ b/deploy/40-statefulset.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: api-server + app.kubernetes.io/name: p5x-api-server + name: p5x-api-server + namespace: p5x-system +spec: + replicas: 1 + selector: + matchLabels: + app: p5x-api-server + serviceName: p5x-api-server + template: + metadata: + labels: + app: p5x-api-server + spec: + priorityClassName: system-cluster-critical + volumes: + - name: p5x-system-data + persistentVolumeClaim: + claimName: system-data + containers: + - name: api-server + image: registry.millslan.net/glmdev/p5x-api:latest +# TODO: (1) add a dedicated /healthz endpoint and (2) prevent that endpoint from writing to the Rocket logs +# livenessProbe: +# failureThreshold: 5 +# httpGet: +# path: /system/pubkey +# port: p5x +# initialDelaySeconds: 10 +# periodSeconds: 120 +# timeoutSeconds: 10 + envFrom: + - configMapRef: + name: api-env + env: + - name: P5X_NODE_HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 3450 + name: p5x + volumeMounts: + - mountPath: /p5x/data + name: p5x-system-data diff --git a/scripts/build.sh b/scripts/build.sh index 6e53126..065785f 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -6,4 +6,4 @@ if ! [ -f Cargo.toml ]; then fi cargo build --release --target x86_64-unknown-linux-gnu -docker build -t "${DOCKER_REGISTRY}/p5x-rs:latest" . +docker build -t "${DOCKER_REGISTRY}/glmdev/p5x-api:latest" . diff --git a/scripts/push.sh b/scripts/push.sh new file mode 100755 index 0000000..fb9adab --- /dev/null +++ b/scripts/push.sh @@ -0,0 +1,3 @@ +#!/bin/sh -e + +docker push "${DOCKER_REGISTRY}/glmdev/p5x-api:latest" diff --git a/scripts/run.sh b/scripts/run.sh index fcb1c08..7deaafe 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -24,4 +24,4 @@ fi echo "" echo "Running p5x..." echo "" -docker run $P5X_ARGS "$DOCKER_REGISTRY/p5x-rs:latest" +docker run $P5X_ARGS "$DOCKER_REGISTRY/glmdev/p5x-api:latest" diff --git a/src/api/db/mod.rs b/src/api/db/mod.rs index cc2bc01..4313363 100644 --- a/src/api/db/mod.rs +++ b/src/api/db/mod.rs @@ -7,7 +7,6 @@ use sea_orm_rocket::{Config, Database, Pool}; use async_trait::async_trait; use rocket::fairing::AdHoc; use sea_orm::ConnectOptions; -use crate::api::entity::locks::ensure_vmid_lock; #[derive(Database, Debug)] #[database("p5x_api")] @@ -43,8 +42,6 @@ impl Pool for DbPool { let conn = sea_orm::Database::connect(options).await?; - ensure_vmid_lock(&conn).await?; // todo: probably a better place to put this - Ok(DbPool { conn }) } diff --git a/src/api/mod.rs b/src/api/mod.rs index 4d95633..082415d 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -5,9 +5,22 @@ mod route; pub mod util; pub mod cluster; pub mod entity; -pub use db::Db; +use sea_orm_rocket::Database; +use db::Db; +use crate::api::entity::locks::ensure_vmid_lock; + pub mod services; +/** Perform any init-time operations that require the DB. */ +pub fn post_init() -> AdHoc { + AdHoc::on_ignite("post_init", |rocket| async { + let conn = &Db::fetch(&rocket).unwrap().conn; + ensure_vmid_lock(&conn).await.unwrap(); + + rocket + }) +} + pub fn init() -> AdHoc { AdHoc::on_ignite("mod(db)", |rocket| async { rocket.attach(db::init())