diff --git a/defaults.inc b/defaults.inc index f71e250..70652dd 100755 --- a/defaults.inc +++ b/defaults.inc @@ -3,7 +3,7 @@ action="$1" FOLDER='/opt/idssys/ta-proxmenu' -VERS='2026.7.25-1' +VERS='2026.7.25-4' noupdate=' ' diff --git a/inc/deploy-proxmox-keepalived.sh b/inc/deploy-proxmox-keepalived.sh new file mode 100644 index 0000000..6cab58e --- /dev/null +++ b/inc/deploy-proxmox-keepalived.sh @@ -0,0 +1,790 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# ----------------------------------------------------------------------------- +# Proxmox cluster Keepalived deployment +# Version 1.5.0 +# +# - Discovers cluster members with `pvecm nodes` +# - Prompts for the floating VIP/CIDR unless supplied on the command line +# - Discovers local Linux bridges and prompts the user to select one unless +# supplied on the command line +# - Selects each node's unicast source address from the VIP's subnet +# - Derives VRRP priority from the hostname's trailing numeric suffix: +# suffix 1 = 200, suffix 2 = 190, suffix 3 = 180, etc. +# - Deploys a Proxmox-aware health check including cluster quorum +# - Uses nopreempt so the VIP does not fail back after a recovered node returns +# - Uses unicast VRRP and validates that exactly one node owns the VIP after the +# VRRP election has had time to settle +# - Starts the highest-priority node first so initial ownership is deterministic +# +# Run this from any healthy, quorate Proxmox cluster node as root. +# ----------------------------------------------------------------------------- + +SCRIPT_VERSION="1.5.0" +VIP_CIDR="" +INTERFACE="" +PRIORITY_BASE=200 +PRIORITY_STEP=10 +VRID=51 +ADVERT_INT=1 +ASSUME_YES=0 + +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout=8 + -o StrictHostKeyChecking=accept-new +) + +usage() { + cat </ --interface vmbr0 + $0 --vip / --interface vmbr1 --vrid 61 -y +USAGE +} + +log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; } +warn() { printf '\nWARNING: %s\n' "$*" >&2; } +die() { printf '\nERROR: %s\n' "$*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --vip) + [[ $# -ge 2 ]] || die "--vip requires a value" + VIP_CIDR="$2"; shift 2 ;; + --interface) + [[ $# -ge 2 ]] || die "--interface requires a value" + INTERFACE="$2"; shift 2 ;; + --vrid) + [[ $# -ge 2 ]] || die "--vrid requires a value" + VRID="$2"; shift 2 ;; + -y|--yes) + ASSUME_YES=1; shift ;; + --version) + echo "deploy-proxmox-keepalived.sh ${SCRIPT_VERSION}"; exit 0 ;; + -h|--help) + usage; exit 0 ;; + *) + die "Unknown option: $1" ;; + esac +done + +[[ $EUID -eq 0 ]] || die "Run this script as root." +command -v pvecm >/dev/null 2>&1 || die "pvecm not found. Run this on a Proxmox VE node." +command -v python3 >/dev/null 2>&1 || die "python3 is required." +command -v ip >/dev/null 2>&1 || die "iproute2 is required." + +printf '\nProxmox Keepalived Cluster Deployment v%s\n' "$SCRIPT_VERSION" +printf '------------------------------------------------\n' + +[[ "$PRIORITY_BASE" =~ ^[0-9]+$ ]] || die "Priority base must be numeric." +[[ "$PRIORITY_STEP" =~ ^[0-9]+$ ]] || die "Priority step must be numeric." +[[ "$VRID" =~ ^[0-9]+$ ]] || die "VRID must be numeric." +(( VRID >= 1 && VRID <= 255 )) || die "VRID must be between 1 and 255." + +validate_vip_cidr() { + local cidr="$1" + python3 - "$cidr" <<'PY' >/dev/null 2>&1 +import ipaddress +import sys + +try: + iface = ipaddress.ip_interface(sys.argv[1]) +except ValueError: + raise SystemExit(1) + +if iface.version != 4: + raise SystemExit(1) + +# A shared Ethernet VIP needs room for the VIP plus at least two real hosts. +if iface.network.prefixlen > 30: + raise SystemExit(1) + +# Do not allow the subnet's network or broadcast address as the VIP. +if iface.ip in (iface.network.network_address, iface.network.broadcast_address): + raise SystemExit(1) + +raise SystemExit(0) +PY +} + +ip_is_in_vip_subnet() { + local ip_addr="$1" + python3 - "$VIP_CIDR" "$ip_addr" <<'PY' >/dev/null 2>&1 +import ipaddress +import sys + +network = ipaddress.ip_interface(sys.argv[1]).network +try: + address = ipaddress.ip_address(sys.argv[2]) +except ValueError: + raise SystemExit(1) +raise SystemExit(0 if address in network else 1) +PY +} + +# Select the Linux bridge first. The VIP is customer-specific, so when the VIP +# is not supplied on the command line we use the selected bridge's *actual* +# addressing to show the operator which network(s) are valid. Nothing here +# assumes a particular customer subnet. +if [[ -z "$INTERFACE" ]]; then + mapfile -t LOCAL_BRIDGES < <( + for bridge_dir in /sys/class/net/*/bridge; do + [[ -d "$bridge_dir" ]] || continue + basename "$(dirname "$bridge_dir")" + done | sort -V + ) + + (( ${#LOCAL_BRIDGES[@]} > 0 )) || die "No Linux bridge interfaces were discovered on this node." + + printf '\nAvailable Linux bridges on %s:\n\n' "$(hostname -s)" + for i in "${!LOCAL_BRIDGES[@]}"; do + bridge="${LOCAL_BRIDGES[$i]}" + ipv4="$(ip -4 -o addr show dev "$bridge" scope global 2>/dev/null | awk '{print $4}' | paste -sd ',' -)" + [[ -n "$ipv4" ]] || ipv4="no IPv4 address" + printf ' %2d) %-16s %s\n' "$((i + 1))" "$bridge" "$ipv4" + done + + while true; do + printf '\n' + read -r -p "Select bridge [1-${#LOCAL_BRIDGES[@]}]: " bridge_choice + if [[ "$bridge_choice" =~ ^[0-9]+$ ]] \ + && (( bridge_choice >= 1 && bridge_choice <= ${#LOCAL_BRIDGES[@]} )); then + INTERFACE="${LOCAL_BRIDGES[$((bridge_choice - 1))]}" + break + fi + printf 'Invalid selection.\n' >&2 + done +fi + +[[ "$INTERFACE" =~ ^[A-Za-z0-9_.:-]+$ ]] || die "Invalid interface name: $INTERFACE" +ip link show "$INTERFACE" >/dev/null 2>&1 || die "Interface ${INTERFACE} does not exist on the local node." +[[ -d "/sys/class/net/${INTERFACE}/bridge" ]] || die "Interface ${INTERFACE} is not a Linux bridge." + +# Read the real IPv4 networks configured on the selected bridge. These are used +# only to guide/validate the user's VIP entry; they are never replaced by a +# hard-coded network. +mapfile -t LOCAL_BRIDGE_CIDRS < <( + ip -4 -o addr show dev "$INTERFACE" scope global 2>/dev/null \ + | awk '{print $4}' \ + | sort -u +) +(( ${#LOCAL_BRIDGE_CIDRS[@]} > 0 )) \ + || die "Selected bridge ${INTERFACE} has no global IPv4 address on the local node." + +mapfile -t LOCAL_BRIDGE_NETWORKS < <( + python3 - "${LOCAL_BRIDGE_CIDRS[@]}" <<'PY' +import ipaddress +import sys +seen = set() +for raw in sys.argv[1:]: + try: + iface = ipaddress.ip_interface(raw) + except ValueError: + continue + network = str(iface.network) + if network not in seen: + seen.add(network) + print(network) +PY +) + +printf '\nSelected bridge: %s\n' "$INTERFACE" +printf 'IPv4 addressing on this node:\n' +for cidr in "${LOCAL_BRIDGE_CIDRS[@]}"; do + network="$(python3 - "$cidr" <<'PY' +import ipaddress, sys +print(ipaddress.ip_interface(sys.argv[1]).network) +PY +)" + printf ' %-18s network %s\n' "$cidr" "$network" +done + +vip_matches_local_bridge() { + local vip_cidr="$1" + shift + python3 - "$vip_cidr" "$@" <<'PY' >/dev/null 2>&1 +import ipaddress +import sys + +try: + vip = ipaddress.ip_interface(sys.argv[1]) +except ValueError: + raise SystemExit(1) + +for raw in sys.argv[2:]: + try: + bridge = ipaddress.ip_interface(raw) + except ValueError: + continue + # Do not allow an already-present VIP itself to satisfy the check on a + # redeploy; there must be a real node address in the same network. + if bridge.ip == vip.ip: + continue + if bridge.network == vip.network: + raise SystemExit(0) +raise SystemExit(1) +PY +} + +# Prompt for the VIP after bridge selection so every customer sees their own +# network information before entering the floating address. +if [[ -z "$VIP_CIDR" ]]; then + while true; do + printf '\n' + read -r -p "Enter the unused floating VIP/CIDR for ${INTERFACE}: " VIP_CIDR + + if ! validate_vip_cidr "$VIP_CIDR"; then + printf 'Invalid IPv4 VIP/CIDR. Enter an unused host address with a /1 through /30 prefix.\n' >&2 + continue + fi + + if ! vip_matches_local_bridge "$VIP_CIDR" "${LOCAL_BRIDGE_CIDRS[@]}"; then + printf 'The VIP must be in one of the selected bridge networks, using the same prefix:\n' >&2 + for network in "${LOCAL_BRIDGE_NETWORKS[@]}"; do + printf ' %s\n' "$network" >&2 + done + continue + fi + + break + done +else + validate_vip_cidr "$VIP_CIDR" || die "Invalid VIP/CIDR: $VIP_CIDR" + vip_matches_local_bridge "$VIP_CIDR" "${LOCAL_BRIDGE_CIDRS[@]}" \ + || die "VIP ${VIP_CIDR} is not on a network configured on ${INTERFACE}. Valid local network(s): ${LOCAL_BRIDGE_NETWORKS[*]}" +fi + +# Derive the bare VIP address only after the customer-specific VIP has been +# validated against the selected bridge. +VIP_IP="$(python3 - "$VIP_CIDR" <<'PY' +import ipaddress +import sys +print(ipaddress.ip_interface(sys.argv[1]).ip) +PY +)" +VIP_PREFIX="${VIP_CIDR#*/}" + +# This check intentionally uses the Proxmox-reported quorum state. +if ! timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'; then + die "The local node does not currently see a quorate cluster. Refusing deployment." +fi + +# Use pmxcfs' JSON membership file for node discovery instead of parsing the +# human-formatted output of `pvecm nodes`. This is important on clusters with a +# QDevice: pvecm adds a Qdevice status column (for example A,V,NMW), which shifts +# the Name column and makes fixed-column awk parsing unsafe. /etc/pve/.members +# contains only real Proxmox cluster members in its nodelist. +MEMBERS_FILE="/etc/pve/.members" +[[ -r "$MEMBERS_FILE" ]] || die "Cannot read ${MEMBERS_FILE}; pmxcfs/cluster membership data is unavailable." + +LOCAL_NODE="$(python3 - "$MEMBERS_FILE" <<'PYMEM' +import json, sys +try: + with open(sys.argv[1], 'r', encoding='utf-8') as f: + data = json.load(f) +except Exception: + raise SystemExit(1) +name = data.get('nodename') +if not isinstance(name, str) or not name: + raise SystemExit(1) +print(name) +PYMEM +)" || die "Unable to determine the local Proxmox node name from ${MEMBERS_FILE}." + +mapfile -t NODES < <(python3 - "$MEMBERS_FILE" <<'PYMEM' +import json, sys +try: + with open(sys.argv[1], 'r', encoding='utf-8') as f: + data = json.load(f) +except Exception: + raise SystemExit(1) +nodelist = data.get('nodelist') +if not isinstance(nodelist, dict): + raise SystemExit(1) +items = [] +for name, info in nodelist.items(): + if not isinstance(name, str) or not name: + continue + node_id = 0 + if isinstance(info, dict): + try: + node_id = int(info.get('id', 0)) + except (TypeError, ValueError): + node_id = 0 + items.append((node_id, name)) +for _, name in sorted(items, key=lambda x: (x[0], x[1])): + print(name) +PYMEM +) + +(( ${#NODES[@]} >= 2 )) || die "Fewer than two Proxmox cluster nodes were found in ${MEMBERS_FILE}." +printf '%s\n' "${NODES[@]}" | grep -Fxq "$LOCAL_NODE" \ + || die "Local node ${LOCAL_NODE} is not present in ${MEMBERS_FILE} nodelist." + +DEPLOY_TAG="$(date +%Y%m%d-%H%M%S)-$$" +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT + +remote_exec() { + local node="$1" + local cmd="$2" + + if [[ "$node" == "$LOCAL_NODE" ]]; then + bash -lc "$cmd" + else + ssh "${SSH_OPTS[@]}" "root@${node}" "$cmd" + fi +} + +copy_to_node() { + local node="$1" + local src="$2" + local dst="$3" + + if [[ "$node" == "$LOCAL_NODE" ]]; then + cp -f "$src" "$dst" + else + scp -q "${SSH_OPTS[@]}" "$src" "root@${node}:${dst}" + fi +} + +# Protect against duplicate node records before using associative arrays. +mapfile -t UNIQUE_NODES < <(printf '%s\n' "${NODES[@]}" | awk 'NF && !seen[$0]++') +(( ${#UNIQUE_NODES[@]} == ${#NODES[@]} )) || die "Duplicate node names were returned by ${MEMBERS_FILE}." +NODES=("${UNIQUE_NODES[@]}") + +declare -A NODE_IP +declare -A NODE_PRIORITY +declare -A PRIORITY_OWNER + +log "Preflight: discovered ${#NODES[@]} Proxmox nodes from ${MEMBERS_FILE}." + +# First pass: verify SSH/access, interface addresses, health, suffixes and priorities. +for node in "${NODES[@]}"; do + if [[ "$node" != "$LOCAL_NODE" ]]; then + ssh "${SSH_OPTS[@]}" "root@${node}" true \ + || die "Passwordless root SSH to ${node} failed. No changes have been made." + fi + + remote_exec "$node" "command -v ip >/dev/null 2>&1 && command -v systemctl >/dev/null 2>&1 && command -v pvecm >/dev/null 2>&1 && command -v timeout >/dev/null 2>&1" \ + || die "${node} is missing one or more required base commands (ip, systemctl, pvecm, timeout)." + + remote_exec "$node" "ip link show '$INTERFACE' >/dev/null 2>&1" \ + || die "Interface ${INTERFACE} does not exist on ${node}." + remote_exec "$node" "test -d '/sys/class/net/${INTERFACE}/bridge'" \ + || die "Interface ${INTERFACE} exists on ${node}, but it is not a Linux bridge." + + # Gather all global IPv4 CIDRs on this bridge, excluding the requested VIP + # so rerunning the installer while Keepalived is active is safe. + mapfile -t candidate_cidrs < <( + remote_exec "$node" \ + "ip -4 -o addr show dev '$INTERFACE' scope global | awk '{print \$4}' | grep -v '^${VIP_IP}/' || true" + ) + + subnet_matches=() + prefix_mismatches=() + for candidate_cidr in "${candidate_cidrs[@]}"; do + [[ "$candidate_cidr" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]{1,2}$ ]] || continue + candidate="${candidate_cidr%/*}" + candidate_prefix="${candidate_cidr#*/}" + + if ip_is_in_vip_subnet "$candidate"; then + if [[ "$candidate_prefix" == "$VIP_PREFIX" ]]; then + subnet_matches+=("$candidate") + else + prefix_mismatches+=("$candidate_cidr") + fi + fi + done + + if (( ${#subnet_matches[@]} == 0 && ${#prefix_mismatches[@]} > 0 )); then + die "${node}:${INTERFACE} has address(es) inside the VIP network but with a different prefix (${prefix_mismatches[*]}). VIP ${VIP_CIDR} must use the same prefix as the management bridge." + fi + + (( ${#subnet_matches[@]} > 0 )) \ + || die "No non-VIP IPv4 address in ${VIP_CIDR} with matching /${VIP_PREFIX} prefix was found on ${node}:${INTERFACE}." + + ip="${subnet_matches[0]}" + if (( ${#subnet_matches[@]} > 1 )); then + warn "${node}:${INTERFACE} has multiple addresses in the VIP subnet (${subnet_matches[*]}). Using ${ip} as the VRRP source." + fi + + [[ "$node" =~ ([0-9]+)$ ]] \ + || die "Hostname '${node}' has no trailing numeric suffix." + + # 10# prevents numbers with leading zeroes from being interpreted as octal. + # Host 1 starts at priority 200, then each suffix step drops by 10: + # *1 = 200, *2 = 190, *3 = 180, etc. + suffix=$((10#${BASH_REMATCH[1]})) + (( suffix >= 1 )) || die "Hostname suffix for ${node} must be 1 or greater." + priority=$((PRIORITY_BASE - ((suffix - 1) * PRIORITY_STEP))) + + (( priority >= 1 && priority <= 254 )) \ + || die "Calculated priority ${priority} for ${node} is outside 1..254. Host suffix is too large for base ${PRIORITY_BASE} with step ${PRIORITY_STEP}." + + if [[ -n "${PRIORITY_OWNER[$priority]:-}" ]]; then + die "Priority collision: ${node} and ${PRIORITY_OWNER[$priority]} both calculate to ${priority}." + fi + + # Verify the remote node itself currently sees quorum before touching it. + remote_exec "$node" \ + "timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'" \ + || die "${node} does not currently report Quorate: Yes. No changes have been made." + + NODE_IP["$node"]="$ip" + NODE_PRIORITY["$node"]="$priority" + PRIORITY_OWNER["$priority"]="$node" +done + +# Prevent accidentally assigning a VIP equal to a real node address. +for node in "${NODES[@]}"; do + [[ "${NODE_IP[$node]}" != "$VIP_IP" ]] \ + || die "VIP ${VIP_IP} is already the selected management address of ${node}." +done + +# Sort nodes by descending priority. This matters with nopreempt: start the +# highest-priority node first so it deterministically becomes the initial MASTER. +mapfile -t SORTED_NODES < <( + for node in "${NODES[@]}"; do + printf '%s %s\n' "${NODE_PRIORITY[$node]}" "$node" + done | sort -k1,1nr -k2,2V | awk '{print $2}' +) + +printf '\nDeployment plan:\n' +printf ' VIP: %s\n' "$VIP_CIDR" +printf ' Interface: %s\n' "$INTERFACE" +printf ' VRID: %s\n' "$VRID" +printf ' Priority: suffix 1 = %s, then -%s per suffix\n' "$PRIORITY_BASE" "$PRIORITY_STEP" +printf ' Failback: disabled (nopreempt)\n' +printf ' VRRP mode: unicast, IP protocol 112\n\n' +printf ' %-28s %-16s %-8s\n' "NODE" "MANAGEMENT IP" "PRIORITY" +printf ' %-28s %-16s %-8s\n' "----------------------------" "----------------" "--------" +for node in "${SORTED_NODES[@]}"; do + printf ' %-28s %-16s %-8s\n' "$node" "${NODE_IP[$node]}" "${NODE_PRIORITY[$node]}" +done + +printf '\nIMPORTANT: VRRP uses IP protocol 112, not TCP/UDP. It must be permitted\n' +printf 'between the listed node IPs on the selected bridge/network. The script will\n' +printf 'wait for the election to settle and detect the common failure case where\n' +printf 'blocked VRRP traffic causes multiple nodes to claim the VIP.\n' + +if (( ASSUME_YES == 0 )); then + printf '\nThis will install/reconfigure Keepalived on every listed node.\n' + read -r -p 'Proceed? [y/N] ' answer + [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } +fi + +log "Installing Keepalived/curl and staging configuration on all nodes." + +STAGE_DIR="/run/proxmox-keepalived-deploy-${DEPLOY_TAG}" + +for node in "${NODES[@]}"; do + log "Preparing ${node}" + + remote_exec "$node" \ + "DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y keepalived curl >/dev/null" + + remote_exec "$node" \ + "rm -rf '$STAGE_DIR' && install -d -o root -g root -m 0700 '$STAGE_DIR'" + + # Generate a node-specific health check so the HTTPS probe tests the actual + # management address on the selected bridge instead of assuming pveproxy is + # bound to 127.0.0.1:8006. + health_file="$TMPDIR/check-proxmox-keepalived-${node}.sh" + cat > "$health_file" </dev/null \ + | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*\$' || exit 1 + +# Verify the actual externally-facing GUI/API endpoint is answering on the +# address Keepalived uses for this node. TLS verification is intentionally +# disabled because Proxmox commonly uses its cluster CA/node certificate here. +curl --silent --show-error --fail --insecure \ + --connect-timeout 2 --max-time 3 \ + 'https://${NODE_IP[$node]}:8006/' \ + >/dev/null 2>&1 || exit 1 + +exit 0 +HEALTH + chmod 0750 "$health_file" + bash -n "$health_file" || die "Internal error: generated health script for ${node} is invalid." + + remote_health="$STAGE_DIR/check-proxmox-keepalived.sh" + copy_to_node "$node" "$health_file" "$remote_health" + remote_exec "$node" \ + "chown root:root '$remote_health' && chmod 0750 '$remote_health' && bash -n '$remote_health'" \ + || die "Generated health script failed syntax validation on ${node}." + + # Test the staged script before touching the live health-check path. This is + # important on a redeploy: an already-running Keepalived instance may be + # executing the live script every few seconds. + remote_exec "$node" "$remote_health" \ + || die "Staged health check failed on ${node}. No live Keepalived files have been replaced." + + config_file="$TMPDIR/keepalived-${node}.conf" + + { + cat < "$config_file" + + remote_conf="$STAGE_DIR/keepalived.conf" + remote_validate_conf="$STAGE_DIR/keepalived.validate.conf" + copy_to_node "$node" "$config_file" "$remote_conf" + remote_exec "$node" "chown root:root '$remote_conf' && chmod 0600 '$remote_conf'" + + # Validate the staged config against the Keepalived version installed on the + # actual node without changing the live script/config. The validation copy + # points at the root-only staged health script so enable_script_security is + # exercised too. + remote_exec "$node" \ + "sed 's#/usr/local/sbin/check-proxmox-keepalived.sh#$remote_health#' '$remote_conf' > '$remote_validate_conf' && chown root:root '$remote_validate_conf' && chmod 0600 '$remote_validate_conf' && keepalived -t -f '$remote_validate_conf'" \ + || die "Keepalived rejected the generated config for ${node}. No live Keepalived files have been replaced." +done + +# From this point onward, failures should not leave multiple VRRP masters alive. +stop_keepalived_everywhere() { + local n + for n in "${NODES[@]}"; do + remote_exec "$n" "systemctl stop keepalived" >/dev/null 2>&1 || true + done +} + +activation_die() { + local message="$1" + warn "$message" + warn "Stopping Keepalived on every cluster node to avoid leaving an ambiguous VIP state." + stop_keepalived_everywhere + die "$message" +} + +log "All configurations validated. Resetting Keepalived election state." +stop_keepalived_everywhere + +# If the VIP remains after Keepalived has been stopped everywhere, it is being +# assigned statically or by some other mechanism. Starting VRRP in that state +# would be unsafe. +for node in "${NODES[@]}"; do + if remote_exec "$node" \ + "ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then + die "VIP ${VIP_IP} remains assigned on ${node}:${INTERFACE} after Keepalived was stopped. Remove the static/other VIP assignment before deploying." + fi +done + +log "Installing validated health checks on all nodes." +for node in "${NODES[@]}"; do + remote_exec "$node" \ + "if [[ -f /usr/local/sbin/check-proxmox-keepalived.sh ]]; then cp -a /usr/local/sbin/check-proxmox-keepalived.sh /usr/local/sbin/check-proxmox-keepalived.sh.pre-proxmox-vip.${DEPLOY_TAG}; fi; install -o root -g root -m 0750 '$STAGE_DIR/check-proxmox-keepalived.sh' /usr/local/sbin/check-proxmox-keepalived.sh" \ + || activation_die "Failed to install the health check on ${node}." +done + +# Validate the exact final config now that its referenced live health script +# exists. Do this on every node before replacing any live Keepalived config. +log "Performing final Keepalived configuration validation on all nodes." +for node in "${NODES[@]}"; do + remote_exec "$node" "keepalived -t -f '$STAGE_DIR/keepalived.conf'" \ + || activation_die "Final Keepalived validation failed on ${node}." +done + +log "Installing validated Keepalived configs on all nodes." +for node in "${NODES[@]}"; do + remote_exec "$node" \ + "if [[ -f /etc/keepalived/keepalived.conf ]]; then cp -a /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.pre-proxmox-vip.${DEPLOY_TAG}; fi; install -o root -g root -m 0644 '$STAGE_DIR/keepalived.conf' /etc/keepalived/keepalived.conf; systemctl enable keepalived >/dev/null" \ + || activation_die "Failed to install the Keepalived configuration on ${node}." +done + +# One last direct health execution catches anything that changed between +# staging and activation. Keepalived is still stopped cluster-wide here. +for node in "${NODES[@]}"; do + remote_exec "$node" "/usr/local/sbin/check-proxmox-keepalived.sh" \ + || activation_die "Final health check failed on ${node}; Keepalived remains stopped cluster-wide." +done + +MASTER_NODE="${SORTED_NODES[0]}" + +log "Starting highest-priority node first: ${MASTER_NODE} (${NODE_PRIORITY[$MASTER_NODE]})." +remote_exec "$MASTER_NODE" "systemctl start keepalived" \ + || activation_die "Keepalived failed to start on ${MASTER_NODE}." + +# init_fail requires successful health checks before the VRRP instance becomes +# eligible. Allow enough time for rise=3, the VRRP master-down timer and normal +# scheduling jitter. +master_ready=0 +for _ in {1..20}; do + if remote_exec "$MASTER_NODE" \ + "ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then + master_ready=1 + break + fi + sleep 1 +done + +(( master_ready == 1 )) \ + || activation_die "${MASTER_NODE} did not acquire VIP ${VIP_IP}. Check: journalctl -u keepalived" + +for node in "${SORTED_NODES[@]:1}"; do + log "Starting backup node ${node}." + remote_exec "$node" "systemctl start keepalived" \ + || activation_die "Keepalived failed to start on backup node ${node}." +done + +# Do not verify ownership immediately. If VRRP protocol 112 is blocked, every +# isolated BACKUP can take several seconds to decide the MASTER is absent and +# promote itself. Waiting past the normal master-down interval turns that common +# firewall/multicast-unicast failure into a detectable multiple-owner condition. +SETTLE_SECONDS=$((ADVERT_INT * 5 + 2)) +log "Waiting ${SETTLE_SECONDS}s for the VRRP election to settle." +sleep "$SETTLE_SECONDS" + +log "Verifying VIP ownership and service state." + +owners=() +for node in "${NODES[@]}"; do + remote_exec "$node" "systemctl is-active --quiet keepalived" \ + || activation_die "Keepalived is not active on ${node}." + + if remote_exec "$node" \ + "ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then + owners+=("$node") + fi +done + +(( ${#owners[@]} == 1 )) \ + || activation_die "Expected exactly one VIP owner after the election settled, found ${#owners[@]}: ${owners[*]:-none}. Verify IP protocol 112 is allowed between all node IPs and that VRID ${VRID} is not conflicting on this L2 network." + +OWNER_NODE="${owners[0]}" + +# Verify the VIP itself answers the Proxmox API from the deployment node. +if ! curl --silent --show-error --fail --insecure \ + --connect-timeout 2 --max-time 5 \ + "https://${VIP_IP}:8006/" >/dev/null 2>&1; then + warn "VIP ${VIP_IP} is assigned to ${OWNER_NODE}, but the HTTPS test through the VIP failed." + warn "Check routing/firewall rules and any custom pveproxy LISTEN_IP configuration." +else + log "VIP HTTPS test succeeded." +fi + +# Staging lives under /run and is no longer needed after successful activation. +for node in "${NODES[@]}"; do + remote_exec "$node" "rm -rf '$STAGE_DIR'" >/dev/null 2>&1 || true +done + +cat < pve2 > pve3 when PRIORITY_BASE=200 -# - Deploys a Proxmox-aware health check including cluster quorum -# - Uses nopreempt so the VIP does not fail back after a recovered node returns -# - Starts the highest-priority node first so initial ownership is deterministic -# -# Run this from any healthy, quorate Proxmox cluster node as root. -# ----------------------------------------------------------------------------- - -VIP_CIDR="10.2.1.10/24" -INTERFACE="vmbr0" -PRIORITY_BASE=200 -VRID=51 -ADVERT_INT=1 -ASSUME_YES=0 - -SSH_OPTS=( - -o BatchMode=yes - -o ConnectTimeout=8 - -o StrictHostKeyChecking=accept-new -) - -usage() { - cat <&2; } -die() { printf '\nERROR: %s\n' "$*" >&2; exit 1; } - -while [[ $# -gt 0 ]]; do - case "$1" in - --vip) - [[ $# -ge 2 ]] || die "--vip requires a value" - VIP_CIDR="$2"; shift 2 ;; - --interface) - [[ $# -ge 2 ]] || die "--interface requires a value" - INTERFACE="$2"; shift 2 ;; - --priority-base) - [[ $# -ge 2 ]] || die "--priority-base requires a value" - PRIORITY_BASE="$2"; shift 2 ;; - --vrid) - [[ $# -ge 2 ]] || die "--vrid requires a value" - VRID="$2"; shift 2 ;; - -y|--yes) - ASSUME_YES=1; shift ;; - -h|--help) - usage; exit 0 ;; - *) - die "Unknown option: $1" ;; - esac -done - -[[ $EUID -eq 0 ]] || die "Run this script as root." -command -v pvecm >/dev/null 2>&1 || die "pvecm not found. Run this on a Proxmox VE node." -command -v python3 >/dev/null 2>&1 || die "python3 is required." - -[[ "$INTERFACE" =~ ^[A-Za-z0-9_.:-]+$ ]] || die "Invalid interface name: $INTERFACE" -[[ "$PRIORITY_BASE" =~ ^[0-9]+$ ]] || die "Priority base must be numeric." -[[ "$VRID" =~ ^[0-9]+$ ]] || die "VRID must be numeric." -(( VRID >= 1 && VRID <= 255 )) || die "VRID must be between 1 and 255." - -# Validate VIP/CIDR and derive the bare IP. -VIP_IP="$(python3 - "$VIP_CIDR" <<'PY' -import ipaddress, sys -try: - iface = ipaddress.ip_interface(sys.argv[1]) -except ValueError as exc: - print(exc, file=sys.stderr) - raise SystemExit(1) -if iface.version != 4: - print("Only IPv4 VIPs are supported by this script.", file=sys.stderr) - raise SystemExit(1) -print(iface.ip) -PY -)" || die "Invalid VIP/CIDR: $VIP_CIDR" - -# This check intentionally uses the Proxmox-reported quorum state. -if ! timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'; then - die "The local node does not currently see a quorate cluster. Refusing deployment." -fi - -LOCAL_NODE="$(pvecm nodes | awk '$NF == "(local)" {print $3; exit}')" -[[ -n "$LOCAL_NODE" ]] || LOCAL_NODE="$(hostname -s)" -DEPLOY_TAG="$(date +%Y%m%d-%H%M%S)-$$" -TMPDIR="$(mktemp -d)" -trap 'rm -rf "$TMPDIR"' EXIT - -remote_exec() { - local node="$1" - local cmd="$2" - - if [[ "$node" == "$LOCAL_NODE" ]]; then - bash -lc "$cmd" - else - ssh "${SSH_OPTS[@]}" "root@${node}" "$cmd" - fi -} - -copy_to_node() { - local node="$1" - local src="$2" - local dst="$3" - - if [[ "$node" == "$LOCAL_NODE" ]]; then - cp -f "$src" "$dst" - else - scp -q "${SSH_OPTS[@]}" "$src" "root@${node}:${dst}" - fi -} - -# Discover cluster node names. pvecm nodes emits: Nodeid Votes Name [(local)] -mapfile -t NODES < <(pvecm nodes | awk '$1 ~ /^(0x)?[0-9A-Fa-f]+$/ && $2 ~ /^[0-9]+$/ {print $3}') -(( ${#NODES[@]} >= 2 )) || die "Fewer than two cluster nodes were discovered." - -declare -A NODE_IP -declare -A NODE_PRIORITY -declare -A PRIORITY_OWNER - -log "Preflight: discovered ${#NODES[@]} cluster nodes." - -# First pass: verify SSH/access, interface addresses, health, suffixes and priorities. -for node in "${NODES[@]}"; do - if [[ "$node" != "$LOCAL_NODE" ]]; then - ssh "${SSH_OPTS[@]}" "root@${node}" true \ - || die "Passwordless root SSH to ${node} failed. No changes have been made." - fi - - remote_exec "$node" "ip link show '$INTERFACE' >/dev/null 2>&1" \ - || die "Interface ${INTERFACE} does not exist on ${node}." - - # Exclude the VIP so the script remains safe/idempotent if Keepalived is - # already running and this deployment is being refreshed. - ip="$(remote_exec "$node" \ - "ip -4 -o addr show dev '$INTERFACE' scope global | awk '{split(\$4,a,\"/\"); print a[1]}' | grep -vx '$VIP_IP' | head -n1")" - - [[ -n "$ip" ]] || die "Could not determine a non-VIP IPv4 address on ${node}:${INTERFACE}." - [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || die "Unexpected IPv4 address '${ip}' on ${node}." - - # Verify each management address resides inside the VIP's subnet. - python3 - "$VIP_CIDR" "$ip" <<'PY' >/dev/null || die "${node} (${ip}) is not in the VIP subnet ${VIP_CIDR}." -import ipaddress, sys -vip = ipaddress.ip_interface(sys.argv[1]) -ip = ipaddress.ip_address(sys.argv[2]) -raise SystemExit(0 if ip in vip.network else 1) -PY - - [[ "$node" =~ ([0-9]+)$ ]] \ - || die "Hostname '${node}' has no trailing numeric suffix." - - # 10# prevents numbers with leading zeroes from being interpreted as octal. - suffix=$((10#${BASH_REMATCH[1]})) - priority=$((PRIORITY_BASE - suffix)) - - (( priority >= 1 && priority <= 254 )) \ - || die "Calculated priority ${priority} for ${node} is outside 1..254. Adjust --priority-base." - - if [[ -n "${PRIORITY_OWNER[$priority]:-}" ]]; then - die "Priority collision: ${node} and ${PRIORITY_OWNER[$priority]} both calculate to ${priority}." - fi - - # Verify the remote node itself currently sees quorum before touching it. - remote_exec "$node" \ - "timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'" \ - || die "${node} does not currently report Quorate: Yes. No changes have been made." - - NODE_IP["$node"]="$ip" - NODE_PRIORITY["$node"]="$priority" - PRIORITY_OWNER["$priority"]="$node" -done - -# Prevent accidentally assigning a VIP equal to a real node address. -for node in "${NODES[@]}"; do - [[ "${NODE_IP[$node]}" != "$VIP_IP" ]] \ - || die "VIP ${VIP_IP} is already the static management address of ${node}." -done - -# Sort nodes by descending priority. This matters with nopreempt: start the -# highest-priority node first so it deterministically becomes the initial MASTER. -mapfile -t SORTED_NODES < <( - for node in "${NODES[@]}"; do - printf '%s %s\n' "${NODE_PRIORITY[$node]}" "$node" - done | sort -nr | awk '{print $2}' -) - -printf '\nDeployment plan:\n' -printf ' VIP: %s\n' "$VIP_CIDR" -printf ' Interface: %s\n' "$INTERFACE" -printf ' VRID: %s\n' "$VRID" -printf ' Failback: disabled (nopreempt)\n\n' -printf ' %-28s %-16s %-8s\n' "NODE" "MANAGEMENT IP" "PRIORITY" -printf ' %-28s %-16s %-8s\n' "----------------------------" "----------------" "--------" -for node in "${SORTED_NODES[@]}"; do - printf ' %-28s %-16s %-8s\n' "$node" "${NODE_IP[$node]}" "${NODE_PRIORITY[$node]}" -done - -if (( ASSUME_YES == 0 )); then - printf '\nThis will install/reconfigure Keepalived on every listed node.\n' - read -r -p 'Proceed? [y/N] ' answer - [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } -fi - -HEALTH_FILE="$TMPDIR/check-proxmox-keepalived.sh" -cat > "$HEALTH_FILE" <<'HEALTH' -#!/usr/bin/env bash -set -u - -# A node may own the management VIP only if the Proxmox management stack, -# Corosync and cluster quorum are all healthy. - -systemctl is-active --quiet pveproxy || exit 1 -systemctl is-active --quiet pvedaemon || exit 1 -systemctl is-active --quiet pve-cluster || exit 1 -systemctl is-active --quiet corosync || exit 1 - -# A surviving but non-quorate node is intentionally ineligible for the VIP. -timeout 3 pvecm status 2>/dev/null \ - | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$' || exit 1 - -# Verify the actual GUI/API endpoint is answering locally. -curl --silent --show-error --fail --insecure --max-time 3 \ - https://127.0.0.1:8006/ \ - >/dev/null 2>&1 || exit 1 - -exit 0 -HEALTH -chmod 0750 "$HEALTH_FILE" - -log "Installing Keepalived/curl and staging configuration on all nodes." - -for node in "${NODES[@]}"; do - log "Preparing ${node}" - - remote_exec "$node" \ - "DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y keepalived curl >/dev/null" - - remote_health="/tmp/check-proxmox-keepalived.${DEPLOY_TAG}" - copy_to_node "$node" "$HEALTH_FILE" "$remote_health" - remote_exec "$node" \ - "install -o root -g root -m 0750 '$remote_health' /usr/local/sbin/check-proxmox-keepalived.sh && rm -f '$remote_health'" - - config_file="$TMPDIR/keepalived-${node}.conf" - - { - cat < "$config_file" - - remote_conf="/tmp/keepalived.conf.${DEPLOY_TAG}" - copy_to_node "$node" "$config_file" "$remote_conf" - - # Validate before replacing the live config. - remote_exec "$node" "keepalived -t -f '$remote_conf'" \ - || die "Keepalived rejected the generated config for ${node}. Existing config was not replaced." - - remote_exec "$node" \ - "if [[ -f /etc/keepalived/keepalived.conf ]]; then cp -a /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.pre-proxmox-vip.${DEPLOY_TAG}; fi; install -o root -g root -m 0644 '$remote_conf' /etc/keepalived/keepalived.conf; rm -f '$remote_conf'; systemctl enable keepalived >/dev/null" - - # Confirm the health script itself is currently passing. - remote_exec "$node" "/usr/local/sbin/check-proxmox-keepalived.sh" \ - || die "Health check failed on ${node}; Keepalived has not been restarted yet." -done - -log "All configurations validated. Resetting Keepalived election state." - -# Stop all instances so a rerun cannot preserve an unintended existing master. -for node in "${NODES[@]}"; do - remote_exec "$node" "systemctl stop keepalived" || true -done - -MASTER_NODE="${SORTED_NODES[0]}" -MASTER_IP="${NODE_IP[$MASTER_NODE]}" - -log "Starting highest-priority node first: ${MASTER_NODE} (${NODE_PRIORITY[$MASTER_NODE]})." -remote_exec "$MASTER_NODE" "systemctl start keepalived" - -# Wait until the preferred initial node actually owns the VIP before starting -# nopreempt backups. This prevents a lower-priority node from winning first. -master_ready=0 -for _ in {1..12}; do - if remote_exec "$MASTER_NODE" \ - "ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then - master_ready=1 - break - fi - sleep 1 -done - -(( master_ready == 1 )) \ - || die "${MASTER_NODE} did not acquire VIP ${VIP_IP}. Keepalived remains stopped on the other nodes. Check: journalctl -u keepalived" - -for node in "${SORTED_NODES[@]:1}"; do - log "Starting backup node ${node}." - remote_exec "$node" "systemctl start keepalived" -done - -log "Verifying VIP ownership and service state." - -owners=() -for node in "${NODES[@]}"; do - remote_exec "$node" "systemctl is-active --quiet keepalived" \ - || die "Keepalived is not active on ${node}." - - if remote_exec "$node" \ - "ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then - owners+=("$node") - fi -done - -(( ${#owners[@]} == 1 )) \ - || die "Expected exactly one VIP owner, found ${#owners[@]}: ${owners[*]:-none}" - -# Verify the VIP itself answers the Proxmox API from the deployment node. -if ! curl --silent --show-error --fail --insecure --max-time 5 \ - "https://${VIP_IP}:8006/" >/dev/null 2>&1; then - warn "VIP ${VIP_IP} is assigned to ${owners[0]}, but the API test through the VIP failed." - warn "Check routing/firewall rules and whether pveproxy has been restricted to a specific LISTEN_IP." -else - log "VIP API test succeeded." -fi - -cat <