pve-lab/start-lab.sh

#!/usr/bin/env bash
# Bring the lab back up after a host reboot. Assumes the VMs, network, and
# storage pools were already created by a prior 00-07 run — this does NOT
# (re)install anything, it just re-establishes libvirt/networking state that
# doesn't survive a reboot (modular daemon sockets, the NAT/forward iptables
# rules) and starts the existing VMs.
set -euo pipefail
source "$(dirname "$0")/lab.env"

sudo -v   # prompt for the sudo password once, up front

# Re-run the (idempotent) daemon + pool + network setup. Safe to do before any
# VM is started: 01-network.sh recreates the pve-net bridge, which would orphan
# any already-running VM's tap device — so this must happen first, while all
# VMs are still shut off.
echo "==> Re-establishing libvirt daemons, storage pools, and pve-net (with NAT rules)"
"${SCRIPT_DIR}/00-prereqs.sh"
"${SCRIPT_DIR}/01-network.sh"

start_vm() {
  local name="$1"
  local state
  state=$(sudo virsh domstate "$name" 2>/dev/null || echo "missing")
  case "$state" in
    running)    echo "    $name already running" ;;
    "shut off") echo "    starting $name"; sudo virsh start "$name" ;;
    paused)     echo "    resuming $name"; sudo virsh resume "$name" ;;
    missing)    echo "    ERROR: $name is not defined in libvirt"; return 1 ;;
    *)          echo "    $name is '$state'; leaving as-is" ;;
  esac
}

wait_ssh() {
  local ip="$1" name="$2" tries=0
  echo "==> Waiting for $name ($ip) over SSH"
  while ! lab_ssh "$ip" true >/dev/null 2>&1; do
    sleep 5
    tries=$((tries+1))
    if (( tries % 12 == 0 )); then echo "    still waiting for $name ... (${tries}0s)"; fi
    if (( tries > 60 )); then echo "    WARN: $name not reachable after 5m; check 'virsh console $name'"; return 1; fi
  done
  echo "    $name is up"
}

# --- storage VM first, and fully confirmed serving, before either PVE node ---
# The PVE nodes log into the iSCSI portal via pvestatd at boot (node.startup is
# 'manual', so open-iscsi itself won't auto-relogin — PVE drives this). Guest
# VMs configured with onboot=1 are started by 'pve-guests.service', a one-shot
# unit with NO retry: if it fires before the iSCSI/NFS storage is reachable,
# those VMs just fail to start and stay stopped. Starting pvestore first (and
# confirming it's actually serving, not just booted) closes that race.
echo "==> Starting storage VM (must be ready before the PVE nodes boot)"
start_vm "$STORE_NAME" || { echo "Run the full ./00 through ./07 build first."; exit 1; }
wait_ssh "$STORE_IP" "$STORE_NAME"

echo "==> Confirming NFS + iSCSI are actually serving on $STORE_NAME"
svc_ok=0
for i in $(seq 1 24); do
  svc_ok=$(lab_ssh "$STORE_IP" "systemctl is-active nfs-server tgt 2>/dev/null | grep -c active" || echo 0)
  [[ "${svc_ok:-0}" -ge 2 ]] && break
  sleep 5
done
if [[ "${svc_ok:-0}" -ge 2 ]]; then
  echo "    nfs-server + tgt active"
else
  echo "    WARN: storage services not confirmed active after 2m; check 'lab_ssh $STORE_IP systemctl status nfs-server tgt'"
fi

# --- now the PVE nodes ---
echo "==> Starting PVE nodes"
missing=0
start_vm "$PVE1_NAME" || missing=1
start_vm "$PVE2_NAME" || missing=1
if [[ "$missing" == 1 ]]; then
  echo "One or more VMs don't exist yet — run the full ./00 through ./07 build first."
  exit 1
fi
wait_ssh "$PVE1_IP" "$PVE1_NAME" || true
wait_ssh "$PVE2_IP" "$PVE2_NAME" || true

# --- belt-and-suspenders: force storage active, retry any onboot VM that lost
# the race against pve-guests.service before this script existed / on a slower
# boot. Idempotent and harmless if everything already came up cleanly. ---
echo "==> Reconciling shared storage + retrying any onboot VMs that failed to start"
for ip in "$PVE1_IP" "$PVE2_IP"; do
  lab_ssh "$ip" bash -s <<'REMOTE' || echo "    WARN: reconcile failed on $ip"
set -uo pipefail
iscsiadm -m node --loginall=all >/dev/null 2>&1 || true
vgchange -ay pve_shared >/dev/null 2>&1 || true
for id in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
  onboot=$(qm config "$id" 2>/dev/null | awk -F': ' '/^onboot/{print $2}')
  [[ "$onboot" == "1" ]] || continue
  status=$(qm status "$id" 2>/dev/null | awk '{print $2}')
  if [[ "$status" == "stopped" ]]; then
    echo "  starting VM $id on $(hostname) (onboot=1, was stopped)"
    qm start "$id" || echo "  WARN: failed to start VM $id"
  fi
done
REMOTE
done

echo "==> Cluster status"
lab_ssh "$PVE1_IP" "pvecm status" 2>/dev/null || echo "    (pve1 not reachable yet; re-run this script or check the VM console)"

echo
echo "Lab is up. Web UIs: https://${PVE1_IP}:8006  https://${PVE2_IP}:8006"
echo "Run ./cluster-info.sh to refresh CLUSTER-INFO.md, or ./verify.sh for a full check."