garrettmills/pve-lab
1
0
Fork 0
Code Activity

Fix onboot VM startup race with iSCSI storage in start-lab.sh

Browse Source 
The storage VM was started last, but PVE's guest autostart
(pve-guests.service) is a one-shot unit with no retry: if it fires before
pvestatd manages to log into the iSCSI portal (node.startup=manual, so
open-iscsi won't auto-relogin on its own), any onboot VM on lab-lvm just
stays stopped. Start pvestore first and confirm NFS/iSCSI are actually
serving before booting the PVE nodes, then force iSCSI login + LVM
activation and retry any still-stopped onboot VMs as a safety net.
This commit is contained in:
Garrett Mills
2026-08-19 21:10:21 -05:00
parent 498675d4db
commit 73861412be
1 changed files with 63 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
start-lab.sh
View File

@@ -17,22 +17,18 @@ echo "==> Re-establishing libvirt daemons, storage pools, and pve-net (with NAT
"${SCRIPT_DIR}/00-prereqs.sh"
"${SCRIPT_DIR}/01-network.sh"
echo "==> Starting VMs"
missing=0
for name in "$PVE1_NAME" "$PVE2_NAME" "$STORE_NAME"; do
start_vm() {
local name="$1"
local state
state=$(sudo virsh domstate "$name" 2>/dev/null || echo "missing")
case "$state" in
running) echo " $name already running" ;;
"shut off") echo " starting $name"; sudo virsh start "$name" ;;
paused) echo " resuming $name"; sudo virsh resume "$name" ;;
missing) echo " ERROR: $name is not defined in libvirt"; missing=1 ;;
missing) echo " ERROR: $name is not defined in libvirt"; return 1 ;;
*) echo " $name is '$state'; leaving as-is" ;;
esac
done
if [[ "$missing" == 1 ]]; then
echo "One or more VMs don't exist yet — run the full ./00 through ./07 build first."
exit 1
fi
}
wait_ssh() {
local ip="$1" name="$2" tries=0
@@ -45,9 +41,63 @@ wait_ssh() {
done
echo " $name is up"
}
# --- storage VM first, and fully confirmed serving, before either PVE node ---
# The PVE nodes log into the iSCSI portal via pvestatd at boot (node.startup is
# 'manual', so open-iscsi itself won't auto-relogin — PVE drives this). Guest
# VMs configured with onboot=1 are started by 'pve-guests.service', a one-shot
# unit with NO retry: if it fires before the iSCSI/NFS storage is reachable,
# those VMs just fail to start and stay stopped. Starting pvestore first (and
# confirming it's actually serving, not just booted) closes that race.
echo "==> Starting storage VM (must be ready before the PVE nodes boot)"
start_vm "$STORE_NAME" || { echo "Run the full ./00 through ./07 build first."; exit 1; }
wait_ssh "$STORE_IP" "$STORE_NAME"
echo "==> Confirming NFS + iSCSI are actually serving on $STORE_NAME"
svc_ok=0
for i in $(seq 1 24); do
svc_ok=$(lab_ssh "$STORE_IP" "systemctl is-active nfs-server tgt 2>/dev/null | grep -c active" || echo 0)
[[ "${svc_ok:-0}" -ge 2 ]] && break
sleep 5
done
if [[ "${svc_ok:-0}" -ge 2 ]]; then
echo " nfs-server + tgt active"
else
echo " WARN: storage services not confirmed active after 2m; check 'lab_ssh $STORE_IP systemctl status nfs-server tgt'"
fi
# --- now the PVE nodes ---
echo "==> Starting PVE nodes"
missing=0
start_vm "$PVE1_NAME" || missing=1
start_vm "$PVE2_NAME" || missing=1
if [[ "$missing" == 1 ]]; then
echo "One or more VMs don't exist yet — run the full ./00 through ./07 build first."
exit 1
fi
wait_ssh "$PVE1_IP" "$PVE1_NAME" || true
wait_ssh "$PVE2_IP" "$PVE2_NAME" || true
wait_ssh "$STORE_IP" "$STORE_NAME" || true
# --- belt-and-suspenders: force storage active, retry any onboot VM that lost
# the race against pve-guests.service before this script existed / on a slower
# boot. Idempotent and harmless if everything already came up cleanly. ---
echo "==> Reconciling shared storage + retrying any onboot VMs that failed to start"
for ip in "$PVE1_IP" "$PVE2_IP"; do
lab_ssh "$ip" bash -s <<'REMOTE' || echo " WARN: reconcile failed on $ip"
set -uo pipefail
iscsiadm -m node --loginall=all >/dev/null 2>&1 || true
vgchange -ay pve_shared >/dev/null 2>&1 || true
for id in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
onboot=$(qm config "$id" 2>/dev/null | awk -F': ' '/^onboot/{print $2}')
[[ "$onboot" == "1" ]] || continue
status=$(qm status "$id" 2>/dev/null | awk '{print $2}')
if [[ "$status" == "stopped" ]]; then
echo " starting VM $id on $(hostname) (onboot=1, was stopped)"
qm start "$id" || echo " WARN: failed to start VM $id"
fi
done
REMOTE
done
echo "==> Cluster status"
lab_ssh "$PVE1_IP" "pvecm status" 2>/dev/null || echo " (pve1 not reachable yet; re-run this script or check the VM console)"