commit 5f114d9845f747aa7f22175c941888297872c8d2
Author: garrettmills <shout@garrettmills.dev>
Date:   Sun Aug 2 10:40:47 2026 -0500

    Big bang

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..cd25bc1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+CLUSTER-INFO.md
+.ssh
+build
diff --git a/00-prereqs.sh b/00-prereqs.sh
new file mode 100755
index 0000000..ed814b1
--- /dev/null
+++ b/00-prereqs.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Prepare the host: start libvirtd, verify KVM + nested virt, ensure the
+# libvirt working directory exists, and generate the automation SSH key.
+set -euo pipefail
+source "$(dirname "$0")/lab.env"
+
+sudo -v   # prompt for the sudo password once, up front
+
+echo "==> Verifying KVM + nested virtualization"
+[[ -e /dev/kvm ]] || { echo "ERROR: /dev/kvm missing"; exit 1; }
+nested=$(cat /sys/module/kvm_intel/parameters/nested 2>/dev/null \
+       || cat /sys/module/kvm_amd/parameters/nested 2>/dev/null || echo N)
+[[ "$nested" == "Y" || "$nested" == "1" ]] || {
+  echo "ERROR: nested virtualization is not enabled (got '$nested')"; exit 1; }
+echo "    /dev/kvm present, nested=$nested"
+
+echo "==> Enabling libvirt (modular daemons — Fedora default)"
+# Fedora uses split per-driver daemons; the libvirt client connects to a socket
+# per driver (virtqemud for VMs, virtstoraged for pools, virtnetworkd for nets).
+# The deprecated monolithic libvirtd is stood down so the two don't fight over state.
+sudo systemctl disable --now libvirtd.service libvirtd.socket \
+     libvirtd-ro.socket libvirtd-admin.socket 2>/dev/null || true
+for d in virtqemud virtstoraged virtnetworkd virtnodedevd virtsecretd virtnwfilterd virtinterfaced; do
+  sudo systemctl enable --now "${d}.socket"    2>/dev/null || true
+  sudo systemctl enable --now "${d}-ro.socket" 2>/dev/null || true
+done
+sudo virsh -c qemu:///system version >/dev/null
+echo "    libvirt (modular) active"
+
+echo "==> Ensuring default storage pool is running"
+if ! sudo virsh pool-info default >/dev/null 2>&1; then
+  sudo virsh pool-define-as default dir --target /var/lib/libvirt/images
+fi
+sudo virsh pool-start default 2>/dev/null || true
+sudo virsh pool-autostart default 2>/dev/null || true
+
+# Pre-create a persistent pool for our image dir so virt-install reuses it
+# instead of auto-creating a transient one (parallel installs would otherwise
+# race to create the same-named pool and one would fail).
+echo "==> Ensuring 'pve-lab' storage pool for $LIB_DIR"
+if ! sudo virsh pool-info pve-lab >/dev/null 2>&1; then
+  sudo virsh pool-define-as pve-lab dir --target "$LIB_DIR"
+fi
+sudo virsh pool-start pve-lab 2>/dev/null || true
+sudo virsh pool-autostart pve-lab 2>/dev/null || true
+
+echo "==> Creating libvirt working directory: $LIB_DIR"
+sudo install -d -m 0711 "$LIB_DIR"
+# Make sure new files land with the right SELinux label for qemu.
+command -v restorecon >/dev/null && sudo restorecon -R "$LIB_DIR" || true
+
+echo "==> Creating build directory: $BUILD_DIR"
+mkdir -p "$BUILD_DIR"
+
+echo "==> Generating automation SSH key: $SSH_KEY"
+mkdir -p "$(dirname "$SSH_KEY")"
+chmod 700 "$(dirname "$SSH_KEY")"
+if [[ ! -f "$SSH_KEY" ]]; then
+  ssh-keygen -t ed25519 -N "" -C "pve-lab-automation" -f "$SSH_KEY" >/dev/null
+  echo "    created"
+else
+  echo "    already exists, reusing"
+fi
+
+echo
+echo "Prereqs OK. Next: ./01-network.sh"
diff --git a/01-network.sh b/01-network.sh
new file mode 100755
index 0000000..3a24466
--- /dev/null
+++ b/01-network.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Define and start the isolated NAT network "pve-net" with fixed DHCP
+# reservations so each VM always gets the same IP.
+set -euo pipefail
+source "$(dirname "$0")/lab.env"
+
+sudo -v   # prompt for the sudo password once, up front
+
+XML="${BUILD_DIR}/${NET_NAME}.xml"
+cat > "$XML" <<EOF
+<network>
+  <name>${NET_NAME}</name>
+  <forward mode='nat'/>
+  <bridge name='${NET_BRIDGE}' stp='on' delay='0'/>
+  <domain name='${DOMAIN}' localOnly='yes'/>
+  <ip address='${NET_GW}' netmask='255.255.255.0'>
+    <dhcp>
+      <range start='${NET_DHCP_START}' end='${NET_DHCP_END}'/>
+      <host mac='${PVE1_MAC}'  name='${PVE1_NAME}'  ip='${PVE1_IP}'/>
+      <host mac='${PVE2_MAC}'  name='${PVE2_NAME}'  ip='${PVE2_IP}'/>
+      <host mac='${STORE_MAC}' name='${STORE_NAME}' ip='${STORE_IP}'/>
+    </dhcp>
+  </ip>
+</network>
+EOF
+
+echo "==> Defining network ${NET_NAME}"
+if sudo virsh net-info "$NET_NAME" >/dev/null 2>&1; then
+  echo "    already defined; recreating to pick up any changes"
+  sudo virsh net-destroy "$NET_NAME" 2>/dev/null || true
+  sudo virsh net-undefine "$NET_NAME" 2>/dev/null || true
+fi
+sudo virsh net-define "$XML"
+sudo virsh net-autostart "$NET_NAME"
+sudo virsh net-start "$NET_NAME"
+
+echo "==> Network status"
+sudo virsh net-info "$NET_NAME"
+
+# On firewalld hosts, libvirt's NAT network often does not get FORWARD-accept
+# rules, so guests can reach the host gateway but not the internet (apt on the
+# storage VM then fails). Add explicit forward + masquerade for our subnet.
+# Idempotent: '-C' checks for the rule before inserting.
+echo "==> Ensuring host IP forwarding + NAT for ${NET_SUBNET}/${NET_PREFIX} (${NET_BRIDGE})"
+sudo sysctl -wq net.ipv4.ip_forward=1
+sudo iptables -t nat -C POSTROUTING -s "${NET_SUBNET}/${NET_PREFIX}" ! -d "${NET_SUBNET}/${NET_PREFIX}" -j MASQUERADE 2>/dev/null \
+  || sudo iptables -t nat -I POSTROUTING -s "${NET_SUBNET}/${NET_PREFIX}" ! -d "${NET_SUBNET}/${NET_PREFIX}" -j MASQUERADE
+sudo iptables -C FORWARD -i "${NET_BRIDGE}" -j ACCEPT 2>/dev/null || sudo iptables -I FORWARD -i "${NET_BRIDGE}" -j ACCEPT
+sudo iptables -C FORWARD -o "${NET_BRIDGE}" -j ACCEPT 2>/dev/null || sudo iptables -I FORWARD -o "${NET_BRIDGE}" -j ACCEPT
+
+echo
+echo "Network up. Next: ./02-download-iso.sh"
diff --git a/02-download-iso.sh b/02-download-iso.sh
new file mode 100755
index 0000000..610e5c1
--- /dev/null
+++ b/02-download-iso.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Download the Proxmox VE ISO into the libvirt pool dir and verify its sha256.
+set -euo pipefail
+source "$(dirname "$0")/lab.env"
+
+sudo -v   # prompt for the sudo password once, up front
+
+DEST="${LIB_DIR}/${ISO_FILE}"
+
+verify() { echo "${ISO_SHA256}  ${DEST}" | sudo sha256sum -c - >/dev/null 2>&1; }
+
+if sudo test -f "$DEST" && verify; then
+  echo "==> ${ISO_FILE} already present and verified"
+else
+  echo "==> Downloading ${ISO_URL}"
+  sudo curl -fL --progress-bar -o "$DEST" "$ISO_URL"
+  echo "==> Verifying sha256"
+  verify || { echo "ERROR: checksum mismatch for $DEST"; exit 1; }
+  echo "    OK"
+fi
+command -v restorecon >/dev/null && sudo restorecon "$DEST" || true
+echo
+echo "ISO ready. Next: ./03-build-autoinstall-isos.sh"
diff --git a/03-build-autoinstall-isos.sh b/03-build-autoinstall-isos.sh
new file mode 100755
index 0000000..3c167d0
--- /dev/null
+++ b/03-build-autoinstall-isos.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Render the per-node answer files and bake them into bootable
+# "Automated Installation" ISOs using proxmox-auto-install-assistant, which we
+# run inside a debian:trixie container (the tool isn't packaged for Fedora).
+set -euo pipefail
+source "$(dirname "$0")/lab.env"
+
+sudo -v   # prompt for the sudo password once, up front
+
+PUBKEY="$(cat "$SSH_PUB")"
+
+echo "==> Rendering answer files into $BUILD_DIR"
+for n in 1 2; do
+  src="${SCRIPT_DIR}/answer-pve${n}.toml"
+  out="${BUILD_DIR}/answer-pve${n}.toml"
+  sed -e "s|@@ROOT_PASSWORD@@|${ROOT_PASSWORD}|g" \
+      -e "s|@@SSH_PUBKEY@@|${PUBKEY}|g" \
+      "$src" > "$out"
+done
+
+# proxmox-auto-install-assistant needs to read the source ISO and write the
+# output ISO. Stage them in BUILD_DIR (user-writable, bind-mounted into docker),
+# then copy the results into the libvirt pool dir.
+echo "==> Staging source ISO into build dir"
+sudo cp -n "${LIB_DIR}/${ISO_FILE}" "${BUILD_DIR}/${ISO_FILE}"
+sudo chown "$(id -u):$(id -g)" "${BUILD_DIR}/${ISO_FILE}"
+
+echo "==> Building auto-install ISOs in debian:trixie container"
+# NOTE: ':z' relabels the bind mount for SELinux so the container can read it
+# (without it, Fedora's SELinux denies the container access to host files).
+docker run --rm -v "${BUILD_DIR}:/work:z" -w /work debian:trixie bash -euc '
+  export DEBIAN_FRONTEND=noninteractive
+  apt-get update -qq
+  apt-get install -y -qq curl gnupg ca-certificates >/dev/null
+  install -d /etc/apt/keyrings
+  curl -fsSL https://enterprise.proxmox.com/debian/proxmox-release-trixie.gpg \
+    -o /etc/apt/keyrings/proxmox-release-trixie.gpg
+  echo "deb [signed-by=/etc/apt/keyrings/proxmox-release-trixie.gpg] http://download.proxmox.com/debian/pve trixie pve-no-subscription" \
+    > /etc/apt/sources.list.d/pve.list
+  apt-get update -qq
+  apt-get install -y -qq proxmox-auto-install-assistant >/dev/null
+  for n in 1 2; do
+    echo "--- validating answer-pve${n}.toml ---"
+    proxmox-auto-install-assistant validate-answer "/work/answer-pve${n}.toml"
+    echo "--- baking pve${n}-auto.iso ---"
+    rm -f "/work/pve${n}-auto.iso"
+    proxmox-auto-install-assistant prepare-iso "/work/'"${ISO_FILE}"'" \
+      --fetch-from iso \
+      --answer-file "/work/answer-pve${n}.toml" \
+      --output "/work/pve${n}-auto.iso"
+  done
+'
+
+echo "==> Publishing ISOs to ${LIB_DIR}"
+for n in 1 2; do
+  sudo cp -f "${BUILD_DIR}/pve${n}-auto.iso" "${LIB_DIR}/pve${n}-auto.iso"
+done
+command -v restorecon >/dev/null && sudo restorecon -R "$LIB_DIR" || true
+# tidy the staged copy of the big source ISO
+rm -f "${BUILD_DIR}/${ISO_FILE}"
+
+echo
+echo "Auto-install ISOs ready:"
+ls -lh "${LIB_DIR}"/pve1-auto.iso "${LIB_DIR}"/pve2-auto.iso 2>/dev/null || \
+  sudo ls -lh "${LIB_DIR}"/pve1-auto.iso "${LIB_DIR}"/pve2-auto.iso
+echo "Next: ./04-install-nodes.sh"
diff --git a/04-install-nodes.sh b/04-install-nodes.sh
new file mode 100755
index 0000000..f4bb4d0
--- /dev/null
+++ b/04-install-nodes.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Unattended-install both PVE nodes from their auto ISOs, then wait until each
+# is reachable over SSH with the lab key.
+set -euo pipefail
+source "$(dirname "$0")/lab.env"
+
+sudo -v   # prompt for the sudo password once, up front (the two installs run in parallel)
+
+install_node() {
+  local name="$1" mac="$2" ip="$3" log="${BUILD_DIR}/install-${1}.log"
+  local disk="${LIB_DIR}/${name}.qcow2"
+  local iso="${LIB_DIR}/${name}-auto.iso"
+
+  # Already installed and reachable? Leave it alone.
+  if lab_ssh "$ip" true >/dev/null 2>&1; then
+    echo "    $name already up at $ip; skipping install" >&2
+    echo "SKIP"
+    return
+  fi
+
+  if sudo virsh dominfo "$name" >/dev/null 2>&1; then
+    echo "    $name already defined; destroying + undefining first" >&2
+    sudo virsh destroy "$name" 2>/dev/null || true
+    sudo virsh undefine "$name" --remove-all-storage --nvram 2>/dev/null || true
+  fi
+
+  echo "    launching virt-install for $name (log: $log)" >&2
+  # --wait -1 blocks until the installer powers off (reboot-mode=power-off),
+  # at which point virt-install redefines the domain to boot from disk.
+  sudo virt-install \
+    --connect qemu:///system \
+    --name "$name" \
+    --memory "$NODE_RAM_MB" \
+    --vcpus "$NODE_VCPU" \
+    --cpu host-passthrough \
+    --machine q35 \
+    --osinfo detect=on,name=debian12 \
+    --disk "path=${disk},size=${NODE_DISK_GB},format=qcow2,bus=virtio" \
+    --cdrom "$iso" \
+    --network "network=${NET_NAME},mac=${mac},model=virtio" \
+    --graphics vnc,listen=127.0.0.1 \
+    --noautoconsole \
+    --wait -1 >"$log" 2>&1 &
+  echo $!
+}
+
+echo "==> Installing both nodes in parallel"
+p1=$(install_node "$PVE1_NAME" "$PVE1_MAC" "$PVE1_IP")
+sleep 5   # small stagger so the shared storage pool is settled before the 2nd launch
+p2=$(install_node "$PVE2_NAME" "$PVE2_MAC" "$PVE2_IP")
+echo "    virt-install PIDs: pve1=$p1 pve2=$p2"
+
+wait_ssh() {
+  local ip="$1" name="$2" tries=0
+  echo "==> Waiting for $name ($ip) to come up over SSH (this takes several minutes)"
+  while ! lab_ssh "$ip" true >/dev/null 2>&1; do
+    sleep 10
+    tries=$((tries+1))
+    if (( tries % 6 == 0 )); then echo "    still waiting for $name ... (${tries}0s)"; fi
+    if (( tries > 180 )); then echo "ERROR: $name not reachable after 30m"; return 1; fi
+  done
+  echo "    $name is up: $(lab_ssh "$ip" hostname)"
+}
+
+wait_ssh "$PVE1_IP" "$PVE1_NAME"
+wait_ssh "$PVE2_IP" "$PVE2_NAME"
+
+echo
+echo "Both nodes installed and reachable."
+echo "  Web UI: https://${PVE1_IP}:8006  and  https://${PVE2_IP}:8006  (root / \$ROOT_PASSWORD)"
+echo "Next: ./05-storage-vm.sh"
diff --git a/05-storage-vm.sh b/05-storage-vm.sh
new file mode 100755
index 0000000..1f3cc5e
--- /dev/null
+++ b/05-storage-vm.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Provision the dedicated storage VM from a Debian 13 cloud image driven by a
+# cloud-init NoCloud seed. It comes up serving NFS (/srv/nfs) and an iSCSI LUN.
+set -euo pipefail
+source "$(dirname "$0")/lab.env"
+
+sudo -v   # prompt for the sudo password once, up front
+
+PUBKEY="$(cat "$SSH_PUB")"
+BASE_IMG="${LIB_DIR}/${DEBIAN_IMG}"
+OS_DISK="${LIB_DIR}/${STORE_NAME}.qcow2"
+DATA_DISK="${LIB_DIR}/${STORE_NAME}-data.qcow2"
+SEED_ISO="${LIB_DIR}/${STORE_NAME}-seed.iso"
+
+echo "==> Fetching Debian cloud image (if needed)"
+if ! sudo test -f "$BASE_IMG"; then
+  sudo curl -fL --progress-bar -o "$BASE_IMG" "$DEBIAN_URL"
+fi
+
+echo "==> Building cloud-init seed ISO"
+SEED_SRC="${BUILD_DIR}/cloud-init"
+mkdir -p "$SEED_SRC"
+sed -e "s|@@ROOT_PASSWORD@@|${ROOT_PASSWORD}|g" \
+    -e "s|@@SSH_PUBKEY@@|${PUBKEY}|g" \
+    -e "s|@@ISCSI_IQN@@|${ISCSI_IQN}|g" \
+    -e "s|@@ISCSI_LUN_GB@@|${ISCSI_LUN_GB}|g" \
+    "${SCRIPT_DIR}/cloud-init/user-data" > "${SEED_SRC}/user-data"
+cp "${SCRIPT_DIR}/cloud-init/meta-data" "${SEED_SRC}/meta-data"
+genisoimage -quiet -output "${BUILD_DIR}/${STORE_NAME}-seed.iso" \
+  -volid cidata -joliet -rock "${SEED_SRC}/user-data" "${SEED_SRC}/meta-data"
+sudo cp -f "${BUILD_DIR}/${STORE_NAME}-seed.iso" "$SEED_ISO"
+
+echo "==> Preparing disks"
+if sudo virsh dominfo "$STORE_NAME" >/dev/null 2>&1; then
+  echo "    $STORE_NAME already defined; destroying + undefining first"
+  sudo virsh destroy "$STORE_NAME" 2>/dev/null || true
+  sudo virsh undefine "$STORE_NAME" --remove-all-storage 2>/dev/null || true
+fi
+sudo cp -f "$BASE_IMG" "$OS_DISK"
+sudo qemu-img resize "$OS_DISK" 12G
+sudo qemu-img create -f qcow2 "$DATA_DISK" "${STORE_DATA_DISK_GB}G" >/dev/null
+command -v restorecon >/dev/null && sudo restorecon -R "$LIB_DIR" || true
+
+echo "==> Creating storage VM"
+sudo virt-install \
+  --connect qemu:///system \
+  --name "$STORE_NAME" \
+  --memory "$STORE_RAM_MB" \
+  --vcpus "$STORE_VCPU" \
+  --cpu host-passthrough \
+  --machine q35 \
+  --osinfo detect=on,name=debian12 \
+  --import \
+  --disk "path=${OS_DISK},format=qcow2,bus=virtio" \
+  --disk "path=${DATA_DISK},format=qcow2,bus=virtio" \
+  --disk "path=${SEED_ISO},device=cdrom" \
+  --network "network=${NET_NAME},mac=${STORE_MAC},model=virtio" \
+  --graphics vnc,listen=127.0.0.1 \
+  --noautoconsole
+
+echo "==> Waiting for storage VM ($STORE_IP) over SSH"
+tries=0
+while ! lab_ssh "$STORE_IP" true >/dev/null 2>&1; do
+  sleep 10; tries=$((tries+1))
+  if (( tries % 6 == 0 )); then echo "    still waiting ... (${tries}0s)"; fi
+  if (( tries > 90 )); then echo "ERROR: storage VM not reachable after 15m"; exit 1; fi
+done
+
+# count helper: emit a single integer, never fail (grep -c exits 1 on zero match)
+remote_count() { lab_ssh "$STORE_IP" "$1 2>/dev/null | grep -c -- '$2'; true" 2>/dev/null | tr -dc '0-9' | tail -c1; }
+
+echo "==> Verifying services (cloud-init may still be finishing; retrying)"
+nfs_ok=0; iscsi_ok=0
+for i in $(seq 1 30); do
+  nfs_ok=$(remote_count "exportfs -v" "/srv/nfs"); nfs_ok=${nfs_ok:-0}
+  iscsi_ok=$(remote_count "tgtadm --mode target --op show" "${ISCSI_IQN}"); iscsi_ok=${iscsi_ok:-0}
+  if (( nfs_ok >= 1 && iscsi_ok >= 1 )); then break; fi
+  sleep 10
+done
+echo "    NFS export present: ${nfs_ok}, iSCSI target present: ${iscsi_ok}"
+(( nfs_ok >= 1 && iscsi_ok >= 1 )) || {
+  echo "WARN: storage services not confirmed. Check that the VM has internet (apt) and"
+  echo "      'cloud-init status --long' on ${STORE_IP}. See README troubleshooting."; }
+
+echo
+echo "Storage VM ready. Next: ./06-form-cluster.sh"
diff --git a/06-form-cluster.sh b/06-form-cluster.sh
new file mode 100755
index 0000000..21fb80b
--- /dev/null
+++ b/06-form-cluster.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Form the 2-node cluster: create on pve1, join pve2. Sets up /etc/hosts and
+# root SSH trust between the nodes first so pvecm can run non-interactively.
+set -euo pipefail
+source "$(dirname "$0")/lab.env"
+
+HOSTS="10.10.10.11 pve1.${DOMAIN} pve1
+10.10.10.12 pve2.${DOMAIN} pve2
+10.10.10.13 pvestore.${DOMAIN} pvestore"
+
+echo "==> Ensuring /etc/hosts entries on both nodes"
+for ip in "$PVE1_IP" "$PVE2_IP"; do
+  lab_ssh "$ip" "grep -q 'pve1.${DOMAIN}' /etc/hosts || printf '%s\n' \"$HOSTS\" >> /etc/hosts"
+done
+
+echo "==> Establishing root SSH trust pve2 -> pve1 (needed by pvecm add)"
+# Ensure pve2 has a root key, then trust it on pve1; pre-seed known_hosts.
+lab_ssh "$PVE2_IP" "test -f /root/.ssh/id_rsa || ssh-keygen -t rsa -N '' -f /root/.ssh/id_rsa -q"
+PVE2_ROOT_PUB=$(lab_ssh "$PVE2_IP" "cat /root/.ssh/id_rsa.pub")
+lab_ssh "$PVE1_IP" "mkdir -p /root/.ssh && chmod 700 /root/.ssh; \
+  grep -qF '$PVE2_ROOT_PUB' /root/.ssh/authorized_keys 2>/dev/null || echo '$PVE2_ROOT_PUB' >> /root/.ssh/authorized_keys"
+# Pre-seed host keys so pvecm's ssh doesn't choke on prompts.
+lab_ssh "$PVE2_IP" "ssh-keyscan -H ${PVE1_IP} pve1 >> /root/.ssh/known_hosts 2>/dev/null"
+lab_ssh "$PVE1_IP" "ssh-keyscan -H ${PVE2_IP} pve2 >> /root/.ssh/known_hosts 2>/dev/null"
+
+echo "==> Creating cluster '${CLUSTER_NAME}' on pve1"
+if lab_ssh "$PVE1_IP" "pvecm status >/dev/null 2>&1"; then
+  echo "    pve1 already in a cluster; skipping create"
+else
+  lab_ssh "$PVE1_IP" "pvecm create ${CLUSTER_NAME}"
+fi
+
+# corosync reports quorate almost instantly for a single node, but pmxcfs
+# (/etc/pve) takes longer to become WRITABLE after 'create'. 'pvecm addnode' on
+# pve1 needs a writable /etc/pve or it dies "cluster not ready - no quorum?".
+# So wait on an actual write to /etc/pve, not on the corosync quorum flag.
+echo "==> Waiting for pve1 /etc/pve (pmxcfs) to become writable"
+for i in $(seq 1 36); do
+  lab_ssh "$PVE1_IP" "echo ok >/etc/pve/.joincheck 2>/dev/null && rm -f /etc/pve/.joincheck" && break
+  sleep 5
+done
+
+# 'pvecm add' can print an error yet still exit 0, so success must be judged by
+# the actual node count on pve1, and the join retried until it takes.
+pve_nodes() { lab_ssh "$PVE1_IP" "pvecm nodes 2>/dev/null" | grep -cE 'pve[12]' || true; }
+
+echo "==> Joining pve2 to the cluster"
+if [[ "$(pve_nodes)" -ge 2 ]]; then
+  echo "    pve2 already joined"
+else
+  for attempt in 1 2 3 4 5; do
+    lab_ssh "$PVE2_IP" "pvecm add ${PVE1_IP} --use_ssh" || true
+    sleep 5
+    cnt="$(pve_nodes)"
+    if [[ "$cnt" -ge 2 ]]; then echo "    joined: cluster now has $cnt node(s)"; break; fi
+    echo "    join attempt ${attempt}: still $cnt node(s); waiting before retry..."
+    sleep 10
+  done
+fi
+cnt="$(pve_nodes)"
+[[ "$cnt" -ge 2 ]] || { echo "ERROR: pve2 did not join (nodes=$cnt)"; exit 1; }
+
+echo "==> Waiting for quorum"
+for i in $(seq 1 30); do
+  if lab_ssh "$PVE1_IP" "pvecm status 2>/dev/null | grep -q 'Quorate: *Yes'"; then break; fi
+  sleep 5
+done
+
+echo "==> Cluster status"
+lab_ssh "$PVE1_IP" "pvecm status; echo; pvecm nodes"
+echo
+echo "Cluster formed. Next: ./07-add-shared-storage.sh"
diff --git a/07-add-shared-storage.sh b/07-add-shared-storage.sh
new file mode 100755
index 0000000..0c71d7b
--- /dev/null
+++ b/07-add-shared-storage.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Wire the shared storage into the cluster (run once on pve1; datacenter config
+# replicates to pve2 automatically):
+#   - lab-nfs : NFS share from the storage VM
+#   - lab-lvm : LVM volume group built on the shared iSCSI LUN
+set -euo pipefail
+source "$(dirname "$0")/lab.env"
+
+has_storage() { lab_ssh "$PVE1_IP" "pvesm status 2>/dev/null | awk '{print \$1}' | grep -qx '$1'"; }
+
+echo "==> Adding NFS storage 'lab-nfs'"
+if has_storage lab-nfs; then
+  echo "    already present"
+else
+  lab_ssh "$PVE1_IP" "pvesm add nfs lab-nfs --server ${STORE_IP} --export /srv/nfs \
+    --content images,iso,vztmpl,backup,rootdir --options vers=4"
+fi
+
+echo "==> Adding iSCSI base storage 'lab-iscsi'"
+if has_storage lab-iscsi; then
+  echo "    already present"
+else
+  echo "    discoverable targets at ${STORE_IP}:"
+  lab_ssh "$PVE1_IP" "pvesm scan iscsi ${STORE_IP}" || true
+  lab_ssh "$PVE1_IP" "pvesm add iscsi lab-iscsi --portal ${STORE_IP} --target ${ISCSI_IQN} --content none"
+fi
+
+echo "==> Locating the iSCSI block device on pve1"
+DEV=""
+for i in $(seq 1 24); do
+  DEV=$(lab_ssh "$PVE1_IP" "for d in /dev/disk/by-path/*iscsi*-lun-1; do [ -e \"\$d\" ] && readlink -f \"\$d\"; done 2>/dev/null | head -1")
+  [[ -n "$DEV" ]] && break
+  # nudge a session login/rescan
+  lab_ssh "$PVE1_IP" "iscsiadm -m node -l 2>/dev/null; iscsiadm -m session -R 2>/dev/null" || true
+  sleep 5
+done
+[[ -n "$DEV" ]] || { echo "ERROR: iSCSI LUN device did not appear on pve1"; exit 1; }
+echo "    LUN device: $DEV"
+
+echo "==> Creating shared LVM volume group 'pve_shared' on the LUN (pve1)"
+if lab_ssh "$PVE1_IP" "vgs --noheadings -o vg_name 2>/dev/null | grep -qw pve_shared"; then
+  echo "    VG pve_shared already exists"
+else
+  lab_ssh "$PVE1_IP" "pvcreate -ff -y ${DEV} && vgcreate pve_shared ${DEV}"
+fi
+
+echo "==> Adding shared LVM storage 'lab-lvm'"
+if has_storage lab-lvm; then
+  echo "    already present"
+else
+  lab_ssh "$PVE1_IP" "pvesm add lvm lab-lvm --vgname pve_shared --shared 1 --content images,rootdir"
+fi
+
+# make sure pve2 also sees the new VG on the shared LUN
+lab_ssh "$PVE2_IP" "iscsiadm -m session -R 2>/dev/null; pvscan --cache 2>/dev/null; vgscan 2>/dev/null" || true
+
+echo "==> Storage status on pve1"
+lab_ssh "$PVE1_IP" "pvesm status"
+echo
+echo "Shared storage configured. Next: ./verify.sh"
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..dd8662f
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,7 @@
+Copyright © 2026 Garrett Mills
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..4a54197
--- /dev/null
+++ b/README.md
@@ -0,0 +1,79 @@
+# `pve-lab`: Local Proxmox VE test cluster (nested in libvirt/KVM)
+
+Stands up a disposable **2-node Proxmox VE 9.2 cluster** plus a **dedicated
+storage VM** that serves **both NFS and iSCSI**, all as nested VMs on this
+machine via `qemu:///system`. Intended for testing software that drives a
+Proxmox cluster.
+
+```
+pve-net  (NAT 10.10.10.0/24, gw .1)
+├── pve1      10.10.10.11   PVE node   4 vCPU / 8 GB / 48 GB
+├── pve2      10.10.10.12   PVE node   4 vCPU / 8 GB / 48 GB
+└── pvestore  10.10.10.13   Debian 13  2 vCPU / 2 GB  (NFS + iSCSI target)
+```
+
+Shared storage exposed to the cluster:
+- **lab-nfs** — NFS share `/srv/nfs` (images, ISOs, backups, containers)
+- **lab-lvm** — shared LVM volume group `pve_shared` on an iSCSI LUN
+
+## Requirements
+- Linux host with KVM + **nested virtualization** enabled.
+- `libvirt`, `virt-install`, `qemu-img`, `genisoimage`, `docker`, `sshpass`.
+- `sudo` (the `0*`/`teardown` scripts manage the system libvirt instance).
+- Internet access to `*.proxmox.com` and `cloud.debian.org`.
+
+## Usage
+Run in order from this directory:
+```bash
+./00-prereqs.sh                 # libvirtd, KVM checks, SSH key, dirs
+./01-network.sh                 # define + start the pve-net network
+./02-download-iso.sh            # fetch + verify the Proxmox ISO
+./03-build-autoinstall-isos.sh  # bake per-node unattended-install ISOs (docker)
+./04-install-nodes.sh           # unattended-install pve1 + pve2 (several minutes)
+./05-storage-vm.sh              # provision the NFS + iSCSI storage VM
+./06-form-cluster.sh            # create the cluster + join pve2
+./07-add-shared-storage.sh      # wire lab-nfs + lab-lvm into the datacenter
+./verify.sh                     # end-to-end checks incl. a cross-node migration
+```
+
+Config (IPs, sizing, password, ISO version) lives in [`lab.env`](lab.env).
+
+## Access
+- Web UI: <https://10.10.10.11:8006> and <https://10.10.10.12:8006>
+- Login: `root` / value of `ROOT_PASSWORD` in `lab.env`
+- SSH: `ssh -i .ssh/id_lab root@10.10.10.11` (key auto-generated by `00`)
+
+## Reset / rebuild
+```bash
+./teardown.sh             # remove VMs, network, images, build artifacts
+./teardown.sh --keep-iso  # keep the downloaded ISOs for a faster rebuild
+```
+
+## Notes
+- The PVE nodes power off after install (`reboot-mode = "power-off"`) so
+  `virt-install` cleanly hands off to disk-boot; they are then started normally.
+- `proxmox-auto-install-assistant` isn't packaged for Fedora, so `03` runs it
+  inside a `debian:trixie` container against the Proxmox `pve-no-subscription`
+  repo. Docker only needs the user to be in the `docker` group (no sudo).
+
+## Troubleshooting
+- **Storage VM can't install packages / no internet in guests.** On firewalld
+  hosts, libvirt's NAT network frequently lacks FORWARD-accept rules, so guests
+  reach the host gateway (`10.10.10.1`) but nothing beyond. `01-network.sh` now
+  adds explicit `iptables` forward + masquerade rules for the subnet to fix this;
+  `teardown.sh` removes them. The storage VM's apt is also forced to IPv4
+  (mirrors resolve to unreachable IPv6 on this NAT). Verify from the host:
+  `ssh -i .ssh/id_lab root@10.10.10.13 ping -c2 1.1.1.1`.
+- **Never run `virsh net-destroy` on a network with running VMs.** It tears down
+  the bridge and orphans the VMs' tap interfaces; you must then restart the
+  domains (`virsh destroy <dom> && virsh start <dom>`) to re-attach them.
+
+## License & genAI
+
+This project is licensed under the terms of the MIT license. See `LICENSE` for
+details.
+
+I believe it is important to disclose when generative AI ("genAI") was used in
+the creation of a work, as is the case here. Much of this was authored using
+Claude Sonnet 5, though I have reviewed and tested by hand.
+
diff --git a/answer-pve1.toml b/answer-pve1.toml
new file mode 100644
index 0000000..00e9c90
--- /dev/null
+++ b/answer-pve1.toml
@@ -0,0 +1,18 @@
+# Proxmox VE 9 automated-install answer file for node pve1.
+# Placeholders (@@...@@) are filled in by 03-build-autoinstall-isos.sh.
+[global]
+keyboard      = "en-us"
+country       = "us"
+fqdn          = "pve1.lab.local"
+mailto        = "root@lab.local"
+timezone      = "UTC"
+root-password = "@@ROOT_PASSWORD@@"
+root-ssh-keys = ["@@SSH_PUBKEY@@"]
+reboot-mode   = "power-off"           # power off after install so virt-install hands off to disk-boot
+
+[network]
+source = "from-dhcp"                  # libvirt DHCP reservation pins this to 10.10.10.11
+
+[disk-setup]
+filesystem = "ext4"
+disk-list  = ["vda"]                  # virtio-blk disk created by virt-install
diff --git a/answer-pve2.toml b/answer-pve2.toml
new file mode 100644
index 0000000..e0d3c2d
--- /dev/null
+++ b/answer-pve2.toml
@@ -0,0 +1,18 @@
+# Proxmox VE 9 automated-install answer file for node pve2.
+# Placeholders (@@...@@) are filled in by 03-build-autoinstall-isos.sh.
+[global]
+keyboard      = "en-us"
+country       = "us"
+fqdn          = "pve2.lab.local"
+mailto        = "root@lab.local"
+timezone      = "UTC"
+root-password = "@@ROOT_PASSWORD@@"
+root-ssh-keys = ["@@SSH_PUBKEY@@"]
+reboot-mode   = "power-off"           # power off after install so virt-install hands off to disk-boot
+
+[network]
+source = "from-dhcp"                  # libvirt DHCP reservation pins this to 10.10.10.12
+
+[disk-setup]
+filesystem = "ext4"
+disk-list  = ["vda"]                  # virtio-blk disk created by virt-install
diff --git a/cloud-init/meta-data b/cloud-init/meta-data
new file mode 100644
index 0000000..3ad2edd
--- /dev/null
+++ b/cloud-init/meta-data
@@ -0,0 +1,2 @@
+instance-id: pvestore-001
+local-hostname: pvestore
diff --git a/cloud-init/user-data b/cloud-init/user-data
new file mode 100644
index 0000000..d46d654
--- /dev/null
+++ b/cloud-init/user-data
@@ -0,0 +1,67 @@
+#cloud-config
+# Storage VM: serves both NFS and iSCSI to the PVE nodes.
+# @@...@@ placeholders are rendered by 05-storage-vm.sh.
+hostname: pvestore
+fqdn: pvestore.lab.local
+manage_etc_hosts: true
+
+# The lab NAT network is IPv4-only; Debian mirrors resolve to IPv6 too, so force
+# apt onto IPv4 and refresh the index before installing.
+package_update: true
+apt:
+  conf: |
+    Acquire::ForceIPv4 "true";
+
+users:
+  - name: root
+    ssh_authorized_keys:
+      - "@@SSH_PUBKEY@@"
+
+ssh_pwauth: true
+chpasswd:
+  expire: false
+  users:
+    - name: root
+      password: "@@ROOT_PASSWORD@@"
+      type: text
+
+# The second disk (vdb) becomes the home for the NFS tree and the iSCSI LUN.
+disk_setup:
+  /dev/vdb:
+    table_type: gpt
+    layout: true
+    overwrite: true
+fs_setup:
+  - device: /dev/vdb1
+    filesystem: ext4
+    label: data
+    overwrite: true
+mounts:
+  - ["/dev/vdb1", "/srv", "ext4", "defaults,nofail", "0", "2"]
+
+packages:
+  - nfs-kernel-server
+  - tgt
+
+write_files:
+  - path: /etc/exports
+    content: |
+      /srv/nfs 10.10.10.0/24(rw,sync,no_subtree_check,no_root_squash)
+  - path: /etc/tgt/conf.d/lab.conf
+    content: |
+      <target @@ISCSI_IQN@@>
+          backing-store /srv/iscsi/lun0.img
+          initiator-address 10.10.10.11
+          initiator-address 10.10.10.12
+      </target>
+
+runcmd:
+  - [ mkdir, -p, /srv/nfs, /srv/iscsi ]
+  - [ chmod, "0777", /srv/nfs ]
+  # sparse backing file for the iSCSI LUN
+  - [ truncate, -s, "@@ISCSI_LUN_GB@@G", /srv/iscsi/lun0.img ]
+  - [ exportfs, -ra ]
+  - [ systemctl, enable, --now, nfs-server ]
+  - [ systemctl, enable, --now, tgt ]
+  - [ systemctl, restart, tgt ]
+  - [ sh, -c, "tgtadm --mode target --op show > /var/log/tgt-targets.log 2>&1 || true" ]
diff --git a/cluster-info.sh b/cluster-info.sh
new file mode 100755
index 0000000..962cb2c
--- /dev/null
+++ b/cluster-info.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Generate CLUSTER-INFO.md: a concise, current description of the running lab
+# (node IPs, credentials, shared storage, live status) for humans and future LLM
+# agents that need to operate against this cluster.
+# NOTE: contains the root password, so the output is gitignored — do not commit.
+set -uo pipefail
+source "$(dirname "$0")/lab.env"
+
+OUT="${SCRIPT_DIR}/CLUSTER-INFO.md"
+
+# pull live info from pve1; empty if unreachable
+p1() { lab_ssh "$PVE1_IP" "$*" 2>/dev/null; }
+pvecm_nodes="$(p1 pvecm nodes)"
+pvecm_status="$(p1 pvecm status)"
+pvesm_status="$(p1 pvesm status)"
+pve_version="$(p1 pveversion)"
+fp="<key not found>"; [[ -f "$SSH_PUB" ]] && fp="$(ssh-keygen -lf "$SSH_PUB" 2>/dev/null)"
+
+{
+cat <<EOF
+# Local Proxmox VE Test Cluster — CLUSTER-INFO
+
+> Auto-generated by \`cluster-info.sh\` on $(date -Is).
+> **Contains the root password. Gitignored — do not commit or share.**
+
+Disposable, nested (libvirt/KVM) Proxmox VE lab on a Fedora workstation, used to
+test software that drives a Proxmox cluster. Managed by the scripts in this
+\`pve-lab/\` directory (see \`README.md\`).
+
+## Nodes
+| Role | Name | IP | Web UI |
+|------|------|----|--------|
+| PVE node 1 | ${PVE1_NAME} | ${PVE1_IP} | https://${PVE1_IP}:8006 |
+| PVE node 2 | ${PVE2_NAME} | ${PVE2_IP} | https://${PVE2_IP}:8006 |
+| Storage (NFS + iSCSI) | ${STORE_NAME} | ${STORE_IP} | (Debian; no web UI) |
+
+Cluster: **${CLUSTER_NAME}** · Domain: **${DOMAIN}** · Network: ${NET_SUBNET}/${NET_PREFIX} (gw ${NET_GW})
+
+## Credentials
+- **Root password** (all PVE nodes + storage VM): \`${ROOT_PASSWORD}\`
+- **SSH key** (passwordless root everywhere): \`${SSH_KEY}\`
+  - Fingerprint: ${fp}
+- SSH from this host — \`IdentitiesOnly=yes\` is **required** (otherwise the agent's
+  keys exhaust the server's MaxAuthTries before the lab key is offered):
+  \`\`\`bash
+  ssh -i ${SSH_KEY} -o IdentitiesOnly=yes root@${PVE1_IP}
+  # or, from pve-lab/:  source lab.env && lab_ssh ${PVE1_IP} pvecm status
+  \`\`\`
+
+## Shared storage (visible on both nodes)
+- **lab-nfs** — NFS from \`${STORE_IP}:/srv/nfs\` (content: images, iso, vztmpl, backup, rootdir)
+- **lab-lvm** — shared LVM volume group \`pve_shared\` on the iSCSI LUN
+- Storage VM iSCSI target: \`${ISCSI_IQN}\` (portal ${STORE_IP}:3260)
+
+## Live status
+EOF
+
+if [[ -n "$pvecm_nodes" ]]; then
+  printf '### pvecm nodes\n```\n%s\n```\n' "$pvecm_nodes"
+  printf '### pvecm status\n```\n%s\n```\n' "$pvecm_status"
+  printf '### pvesm status\n```\n%s\n```\n' "$pvesm_status"
+  [[ -n "$pve_version" ]] && printf '### pveversion\n```\n%s\n```\n' "$pve_version"
+else
+  echo "_Cluster not reachable over SSH when this file was generated; static config above still applies._"
+fi
+} > "$OUT"
+
+chmod 600 "$OUT"   # it holds the root password
+echo "Wrote $OUT"
diff --git a/lab.env b/lab.env
new file mode 100644
index 0000000..ab4396e
--- /dev/null
+++ b/lab.env
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Shared configuration for the local Proxmox VE test cluster.
+
+# --- cluster identity ---
+CLUSTER_NAME="testlab"
+DOMAIN="lab.local"
+
+# --- libvirt connection (system instance; needs sudo) ---
+export LIBVIRT_DEFAULT_URI="qemu:///system"
+VIRSH="sudo virsh"
+VIRTINSTALL="sudo virt-install"
+
+# --- network ---
+NET_NAME="pve-net"
+NET_BRIDGE="pvebr0"
+NET_SUBNET="10.10.10.0"
+NET_PREFIX="24"
+NET_GW="10.10.10.1"
+NET_DHCP_START="10.10.10.100"
+NET_DHCP_END="10.10.10.200"
+
+# --- nodes (name : ip : mac) ---
+PVE1_NAME="pve1";     PVE1_IP="10.10.10.11";  PVE1_MAC="52:54:00:00:10:11"
+PVE2_NAME="pve2";     PVE2_IP="10.10.10.12";  PVE2_MAC="52:54:00:00:10:12"
+STORE_NAME="pvestore"; STORE_IP="10.10.10.13"; STORE_MAC="52:54:00:00:10:13"
+
+# --- credentials ---
+# Min length 8 for the PVE installer. Change if you like.
+ROOT_PASSWORD="ProxmoxLab123!"
+
+# --- node VM sizing ---
+NODE_VCPU="4"
+NODE_RAM_MB="8192"
+NODE_DISK_GB="48"
+
+# --- storage VM sizing ---
+STORE_VCPU="2"
+STORE_RAM_MB="2048"
+STORE_DATA_DISK_GB="60"      # extra data disk on the storage VM (sparse)
+ISCSI_LUN_GB="50"            # sparse backing file size for the iSCSI LUN
+ISCSI_IQN="iqn.2026-06.local.lab:store.lun0"
+
+# --- Proxmox VE ISO ---
+ISO_VERSION="9.2-1"
+ISO_FILE="proxmox-ve_${ISO_VERSION}.iso"
+ISO_URL="https://enterprise.proxmox.com/iso/${ISO_FILE}"
+ISO_SHA256="4e88fe416df9b527624a175f24c9aa07c714d3332afb1ee3dbf3879573ef2c6c"
+
+# --- Debian cloud image (storage VM base); 13 = trixie ---
+DEBIAN_IMG="debian-13-genericcloud-amd64.qcow2"
+DEBIAN_URL="https://cloud.debian.org/images/cloud/trixie/latest/${DEBIAN_IMG}"
+
+# --- working directories ---
+# Scripts and small artifacts live with the repo; heavy images live in the
+# libvirt pool path so qemu:///system (qemu user + SELinux virt_image_t) can
+# read them without permission/labeling headaches.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LIB_DIR="/var/lib/libvirt/images/pve-lab"     # ISOs, qcow2 disks, seed ISO (sudo-owned)
+BUILD_DIR="${SCRIPT_DIR}/build"               # rendered answer files, logs (user-owned)
+
+# --- automation SSH key (generated by 00-prereqs.sh) ---
+SSH_KEY="${SCRIPT_DIR}/.ssh/id_lab"
+SSH_PUB="${SSH_KEY}.pub"
+
+# SSH helper: connect to a node as root with the lab key, no host-key fuss.
+lab_ssh() {
+  ssh -i "$SSH_KEY" \
+      -o IdentitiesOnly=yes \
+      -o StrictHostKeyChecking=no \
+      -o UserKnownHostsFile=/dev/null \
+      -o ConnectTimeout=8 \
+      -o LogLevel=ERROR \
+      "root@$1" "${@:2}"
+}
diff --git a/teardown.sh b/teardown.sh
new file mode 100755
index 0000000..7f27d84
--- /dev/null
+++ b/teardown.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Destroy the whole lab: VMs (and their disks), the network, and staged images.
+# Use --keep-iso to preserve the downloaded Proxmox/Debian ISOs for a faster rebuild.
+set -uo pipefail
+source "$(dirname "$0")/lab.env"
+
+sudo -v   # prompt for the sudo password once, up front
+
+KEEP_ISO=0
+[[ "${1:-}" == "--keep-iso" ]] && KEEP_ISO=1
+
+echo "==> Destroying VMs"
+for d in "$PVE1_NAME" "$PVE2_NAME" "$STORE_NAME"; do
+  sudo virsh destroy "$d" 2>/dev/null || true
+  sudo virsh undefine "$d" --remove-all-storage --nvram 2>/dev/null \
+    || sudo virsh undefine "$d" --remove-all-storage 2>/dev/null || true
+done
+
+echo "==> Removing host NAT/forward rules for ${NET_SUBNET}/${NET_PREFIX}"
+sudo iptables -t nat -D POSTROUTING -s "${NET_SUBNET}/${NET_PREFIX}" ! -d "${NET_SUBNET}/${NET_PREFIX}" -j MASQUERADE 2>/dev/null || true
+sudo iptables -D FORWARD -i "${NET_BRIDGE}" -j ACCEPT 2>/dev/null || true
+sudo iptables -D FORWARD -o "${NET_BRIDGE}" -j ACCEPT 2>/dev/null || true
+
+echo "==> Destroying network ${NET_NAME}"
+sudo virsh net-destroy "$NET_NAME" 2>/dev/null || true
+sudo virsh net-undefine "$NET_NAME" 2>/dev/null || true
+
+echo "==> Removing staged images"
+sudo rm -f "${LIB_DIR}/${PVE1_NAME}.qcow2" "${LIB_DIR}/${PVE2_NAME}.qcow2" \
+           "${LIB_DIR}/${STORE_NAME}.qcow2" "${LIB_DIR}/${STORE_NAME}-data.qcow2" \
+           "${LIB_DIR}/${STORE_NAME}-seed.iso" \
+           "${LIB_DIR}/pve1-auto.iso" "${LIB_DIR}/pve2-auto.iso"
+if [[ "$KEEP_ISO" == 0 ]]; then
+  sudo rm -f "${LIB_DIR}/${ISO_FILE}" "${LIB_DIR}/${DEBIAN_IMG}"
+fi
+
+echo "==> Removing the 'pve-lab' storage pool"
+sudo virsh pool-destroy pve-lab 2>/dev/null || true
+sudo virsh pool-undefine pve-lab 2>/dev/null || true
+
+echo "==> Cleaning build artifacts"
+rm -rf "${BUILD_DIR}"
+
+echo "Teardown complete."
+[[ "$KEEP_ISO" == 1 ]] && echo "(kept downloaded ISOs in ${LIB_DIR})"
diff --git a/verify.sh b/verify.sh
new file mode 100755
index 0000000..bd4e87f
--- /dev/null
+++ b/verify.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# End-to-end verification of the lab: VMs running, cluster quorate, shared
+# storage active on both nodes, and a real cross-node migration over NFS.
+set -uo pipefail
+source "$(dirname "$0")/lab.env"
+
+pass() { echo "  [PASS] $*"; }
+fail() { echo "  [FAIL] $*"; FAILED=1; }
+FAILED=0
+
+echo "== 1. libvirt domains =="
+sudo virsh list --all | grep -E "pve1|pve2|pvestore" || true
+for d in "$PVE1_NAME" "$PVE2_NAME" "$STORE_NAME"; do
+  st=$(sudo virsh domstate "$d" 2>/dev/null || echo missing)
+  [[ "$st" == "running" ]] && pass "$d running" || fail "$d state=$st"
+done
+
+echo "== 2. Cluster quorum =="
+if lab_ssh "$PVE1_IP" "pvecm status 2>/dev/null | grep -q 'Quorate: *Yes'"; then
+  pass "cluster is quorate"
+  lab_ssh "$PVE1_IP" "pvecm nodes"
+else
+  fail "cluster not quorate"
+fi
+
+echo "== 3. Shared storage active on both nodes =="
+for ip in "$PVE1_IP" "$PVE2_IP"; do
+  for s in lab-nfs lab-lvm; do
+    if lab_ssh "$ip" "pvesm status 2>/dev/null | awk '\$1==\"$s\"{print \$3}' | grep -q active"; then
+      pass "$s active on $ip"
+    else
+      fail "$s not active on $ip"
+    fi
+  done
+done
+
+echo "== 4. Cross-node migration over shared NFS =="
+VMID=9001
+lab_ssh "$PVE1_IP" "qm destroy $VMID --purge 2>/dev/null; true"
+if lab_ssh "$PVE1_IP" "qm create $VMID --name migtest --memory 512 \
+      --scsihw virtio-scsi-pci --scsi0 lab-nfs:1 --net0 virtio,bridge=vmbr0"; then
+  pass "created test VM $VMID with disk on lab-nfs"
+  if lab_ssh "$PVE1_IP" "qm migrate $VMID $PVE2_NAME"; then
+    owner=$(lab_ssh "$PVE2_IP" "qm list 2>/dev/null | awk '\$1==$VMID{print \$1}'")
+    [[ "$owner" == "$VMID" ]] && pass "VM $VMID migrated pve1 -> pve2" || fail "VM not found on pve2 after migrate"
+  else
+    fail "qm migrate failed"
+  fi
+  lab_ssh "$PVE2_IP" "qm destroy $VMID --purge 2>/dev/null; true" || \
+    lab_ssh "$PVE1_IP" "qm destroy $VMID --purge 2>/dev/null; true"
+else
+  fail "could not create test VM"
+fi
+
+echo
+if [[ "$FAILED" == 0 ]]; then
+  echo "ALL CHECKS PASSED"
+  echo "Web UIs: https://${PVE1_IP}:8006  https://${PVE2_IP}:8006  (root / \$ROOT_PASSWORD)"
+else
+  echo "SOME CHECKS FAILED — see [FAIL] lines above"
+  exit 1
+fi