Fix onboot VM startup race with iSCSI storage in start-lab.sh

The storage VM was started last, but PVE's guest autostart
(pve-guests.service) is a one-shot unit with no retry: if it fires before
pvestatd manages to log into the iSCSI portal (node.startup=manual, so
open-iscsi won't auto-relogin on its own), any onboot VM on lab-lvm just
stays stopped. Start pvestore first and confirm NFS/iSCSI are actually
serving before booting the PVE nodes, then force iSCSI login + LVM
activation and retry any still-stopped onboot VMs as a safety net.

start-lab.sh

@@ -17,22 +17,18 @@ echo "==> Re-establishing libvirt daemons, storage pools, and pve-net (with NAT
 "${SCRIPT_DIR}/00-prereqs.sh"
 "${SCRIPT_DIR}/01-network.sh"
 
-echo "==> Starting VMs"
-missing=0
-for name in "$PVE1_NAME" "$PVE2_NAME" "$STORE_NAME"; do
+start_vm() {
+  local name="$1"
+  local state
   state=$(sudo virsh domstate "$name" 2>/dev/null || echo "missing")
   case "$state" in
-    running)   echo "    $name already running" ;;
+    running)    echo "    $name already running" ;;
     "shut off") echo "    starting $name"; sudo virsh start "$name" ;;
-    paused)    echo "    resuming $name"; sudo virsh resume "$name" ;;
-    missing)   echo "    ERROR: $name is not defined in libvirt"; missing=1 ;;
-    *)         echo "    $name is '$state'; leaving as-is" ;;
+    paused)     echo "    resuming $name"; sudo virsh resume "$name" ;;
+    missing)    echo "    ERROR: $name is not defined in libvirt"; return 1 ;;
+    *)          echo "    $name is '$state'; leaving as-is" ;;
   esac
-done
-if [[ "$missing" == 1 ]]; then
-  echo "One or more VMs don't exist yet — run the full ./00 through ./07 build first."
-  exit 1
-fi
+}
 
 wait_ssh() {
   local ip="$1" name="$2" tries=0
@@ -45,9 +41,63 @@ wait_ssh() {
   done
   echo "    $name is up"
 }
+
+# --- storage VM first, and fully confirmed serving, before either PVE node ---
+# The PVE nodes log into the iSCSI portal via pvestatd at boot (node.startup is
+# 'manual', so open-iscsi itself won't auto-relogin — PVE drives this). Guest
+# VMs configured with onboot=1 are started by 'pve-guests.service', a one-shot
+# unit with NO retry: if it fires before the iSCSI/NFS storage is reachable,
+# those VMs just fail to start and stay stopped. Starting pvestore first (and
+# confirming it's actually serving, not just booted) closes that race.
+echo "==> Starting storage VM (must be ready before the PVE nodes boot)"
+start_vm "$STORE_NAME" || { echo "Run the full ./00 through ./07 build first."; exit 1; }
+wait_ssh "$STORE_IP" "$STORE_NAME"
+
+echo "==> Confirming NFS + iSCSI are actually serving on $STORE_NAME"
+svc_ok=0
+for i in $(seq 1 24); do
+  svc_ok=$(lab_ssh "$STORE_IP" "systemctl is-active nfs-server tgt 2>/dev/null | grep -c active" || echo 0)
+  [[ "${svc_ok:-0}" -ge 2 ]] && break
+  sleep 5
+done
+if [[ "${svc_ok:-0}" -ge 2 ]]; then
+  echo "    nfs-server + tgt active"
+else
+  echo "    WARN: storage services not confirmed active after 2m; check 'lab_ssh $STORE_IP systemctl status nfs-server tgt'"
+fi
+
+# --- now the PVE nodes ---
+echo "==> Starting PVE nodes"
+missing=0
+start_vm "$PVE1_NAME" || missing=1
+start_vm "$PVE2_NAME" || missing=1
+if [[ "$missing" == 1 ]]; then
+  echo "One or more VMs don't exist yet — run the full ./00 through ./07 build first."
+  exit 1
+fi
 wait_ssh "$PVE1_IP" "$PVE1_NAME" || true
 wait_ssh "$PVE2_IP" "$PVE2_NAME" || true
-wait_ssh "$STORE_IP" "$STORE_NAME" || true
+
+# --- belt-and-suspenders: force storage active, retry any onboot VM that lost
+# the race against pve-guests.service before this script existed / on a slower
+# boot. Idempotent and harmless if everything already came up cleanly. ---
+echo "==> Reconciling shared storage + retrying any onboot VMs that failed to start"
+for ip in "$PVE1_IP" "$PVE2_IP"; do
+  lab_ssh "$ip" bash -s <<'REMOTE' || echo "    WARN: reconcile failed on $ip"
+set -uo pipefail
+iscsiadm -m node --loginall=all >/dev/null 2>&1 || true
+vgchange -ay pve_shared >/dev/null 2>&1 || true
+for id in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
+  onboot=$(qm config "$id" 2>/dev/null | awk -F': ' '/^onboot/{print $2}')
+  [[ "$onboot" == "1" ]] || continue
+  status=$(qm status "$id" 2>/dev/null | awk '{print $2}')
+  if [[ "$status" == "stopped" ]]; then
+    echo "  starting VM $id on $(hostname) (onboot=1, was stopped)"
+    qm start "$id" || echo "  WARN: failed to start VM $id"
+  fi
+done
+REMOTE
+done
 
 echo "==> Cluster status"
 lab_ssh "$PVE1_IP" "pvecm status" 2>/dev/null || echo "    (pve1 not reachable yet; re-run this script or check the VM console)"