diff --git a/defaults.inc b/defaults.inc index e7f38ec..f71e250 100755 --- a/defaults.inc +++ b/defaults.inc @@ -3,7 +3,7 @@ action="$1" FOLDER='/opt/idssys/ta-proxmenu' -VERS='2026.7.18-3' +VERS='2026.7.25-1' noupdate=' ' diff --git a/proxmenu-scripts.sh b/proxmenu-scripts.sh index a18779b..abb0491 100755 --- a/proxmenu-scripts.sh +++ b/proxmenu-scripts.sh @@ -277,6 +277,438 @@ MAINTENANCE_MODE(){ esac } +INSTALL_KEEPALIVE() { + read -n 1 -p "Are you sure you wish to install Keepalive on all Hosts (Y/n)?" choice + case "$choice" in + [Nn]) echo;; + * ) + echo + + set -Eeuo pipefail + + # ----------------------------------------------------------------------------- + # Proxmox cluster Keepalived deployment + # + # - Discovers cluster members with `pvecm nodes` + # - Uses the first non-VIP IPv4 address on the selected interface as each + # node's Keepalived unicast source address + # - Derives VRRP priority from the hostname's trailing numeric suffix + # priority = PRIORITY_BASE - trailing_number + # so pve1 > pve2 > pve3 when PRIORITY_BASE=200 + # - Deploys a Proxmox-aware health check including cluster quorum + # - Uses nopreempt so the VIP does not fail back after a recovered node returns + # - Starts the highest-priority node first so initial ownership is deterministic + # + # Run this from any healthy, quorate Proxmox cluster node as root. + # ----------------------------------------------------------------------------- + + VIP_CIDR="10.2.1.10/24" + INTERFACE="vmbr0" + PRIORITY_BASE=200 + VRID=51 + ADVERT_INT=1 + ASSUME_YES=0 + + SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout=8 + -o StrictHostKeyChecking=accept-new + ) + + usage() { + cat <&2; } + die() { printf '\nERROR: %s\n' "$*" >&2; exit 1; } + + while [[ $# -gt 0 ]]; do + case "$1" in + --vip) + [[ $# -ge 2 ]] || die "--vip requires a value" + VIP_CIDR="$2"; shift 2 ;; + --interface) + [[ $# -ge 2 ]] || die "--interface requires a value" + INTERFACE="$2"; shift 2 ;; + --priority-base) + [[ $# -ge 2 ]] || die "--priority-base requires a value" + PRIORITY_BASE="$2"; shift 2 ;; + --vrid) + [[ $# -ge 2 ]] || die "--vrid requires a value" + VRID="$2"; shift 2 ;; + -y|--yes) + ASSUME_YES=1; shift ;; + -h|--help) + usage; exit 0 ;; + *) + die "Unknown option: $1" ;; + esac + done + + [[ $EUID -eq 0 ]] || die "Run this script as root." + command -v pvecm >/dev/null 2>&1 || die "pvecm not found. Run this on a Proxmox VE node." + command -v python3 >/dev/null 2>&1 || die "python3 is required." + + [[ "$INTERFACE" =~ ^[A-Za-z0-9_.:-]+$ ]] || die "Invalid interface name: $INTERFACE" + [[ "$PRIORITY_BASE" =~ ^[0-9]+$ ]] || die "Priority base must be numeric." + [[ "$VRID" =~ ^[0-9]+$ ]] || die "VRID must be numeric." + (( VRID >= 1 && VRID <= 255 )) || die "VRID must be between 1 and 255." + + # Validate VIP/CIDR and derive the bare IP. + VIP_IP="$(python3 - "$VIP_CIDR" <<'PY' +import ipaddress, sys +try: + iface = ipaddress.ip_interface(sys.argv[1]) +except ValueError as exc: + print(exc, file=sys.stderr) + raise SystemExit(1) +if iface.version != 4: + print("Only IPv4 VIPs are supported by this script.", file=sys.stderr) + raise SystemExit(1) +print(iface.ip) +PY + )" || die "Invalid VIP/CIDR: $VIP_CIDR" + + # This check intentionally uses the Proxmox-reported quorum state. + if ! timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'; then + die "The local node does not currently see a quorate cluster. Refusing deployment." + fi + + LOCAL_NODE="$(pvecm nodes | awk '$NF == "(local)" {print $3; exit}')" + [[ -n "$LOCAL_NODE" ]] || LOCAL_NODE="$(hostname -s)" + DEPLOY_TAG="$(date +%Y%m%d-%H%M%S)-$$" + TMPDIR="$(mktemp -d)" + trap 'rm -rf "$TMPDIR"' EXIT + + remote_exec() { + local node="$1" + local cmd="$2" + + if [[ "$node" == "$LOCAL_NODE" ]]; then + bash -lc "$cmd" + else + ssh "${SSH_OPTS[@]}" "root@${node}" "$cmd" + fi + } + + copy_to_node() { + local node="$1" + local src="$2" + local dst="$3" + + if [[ "$node" == "$LOCAL_NODE" ]]; then + cp -f "$src" "$dst" + else + scp -q "${SSH_OPTS[@]}" "$src" "root@${node}:${dst}" + fi + } + + # Discover cluster node names. pvecm nodes emits: Nodeid Votes Name [(local)] + mapfile -t NODES < <(pvecm nodes | awk '$1 ~ /^(0x)?[0-9A-Fa-f]+$/ && $2 ~ /^[0-9]+$/ {print $3}') + (( ${#NODES[@]} >= 2 )) || die "Fewer than two cluster nodes were discovered." + + declare -A NODE_IP + declare -A NODE_PRIORITY + declare -A PRIORITY_OWNER + + log "Preflight: discovered ${#NODES[@]} cluster nodes." + + # First pass: verify SSH/access, interface addresses, health, suffixes and priorities. + for node in "${NODES[@]}"; do + if [[ "$node" != "$LOCAL_NODE" ]]; then + ssh "${SSH_OPTS[@]}" "root@${node}" true \ + || die "Passwordless root SSH to ${node} failed. No changes have been made." + fi + + remote_exec "$node" "ip link show '$INTERFACE' >/dev/null 2>&1" \ + || die "Interface ${INTERFACE} does not exist on ${node}." + + # Exclude the VIP so the script remains safe/idempotent if Keepalived is + # already running and this deployment is being refreshed. + ip="$(remote_exec "$node" \ + "ip -4 -o addr show dev '$INTERFACE' scope global | awk '{split(\$4,a,\"/\"); print a[1]}' | grep -vx '$VIP_IP' | head -n1")" + + [[ -n "$ip" ]] || die "Could not determine a non-VIP IPv4 address on ${node}:${INTERFACE}." + [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || die "Unexpected IPv4 address '${ip}' on ${node}." + + # Verify each management address resides inside the VIP's subnet. + python3 - "$VIP_CIDR" "$ip" <<'PY' >/dev/null || die "${node} (${ip}) is not in the VIP subnet ${VIP_CIDR}." +import ipaddress, sys +vip = ipaddress.ip_interface(sys.argv[1]) +ip = ipaddress.ip_address(sys.argv[2]) +raise SystemExit(0 if ip in vip.network else 1) +PY + + [[ "$node" =~ ([0-9]+)$ ]] \ + || die "Hostname '${node}' has no trailing numeric suffix." + + # 10# prevents numbers with leading zeroes from being interpreted as octal. + suffix=$((10#${BASH_REMATCH[1]})) + priority=$((PRIORITY_BASE - suffix)) + + (( priority >= 1 && priority <= 254 )) \ + || die "Calculated priority ${priority} for ${node} is outside 1..254. Adjust --priority-base." + + if [[ -n "${PRIORITY_OWNER[$priority]:-}" ]]; then + die "Priority collision: ${node} and ${PRIORITY_OWNER[$priority]} both calculate to ${priority}." + fi + + # Verify the remote node itself currently sees quorum before touching it. + remote_exec "$node" \ + "timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'" \ + || die "${node} does not currently report Quorate: Yes. No changes have been made." + + NODE_IP["$node"]="$ip" + NODE_PRIORITY["$node"]="$priority" + PRIORITY_OWNER["$priority"]="$node" + done + + # Prevent accidentally assigning a VIP equal to a real node address. + for node in "${NODES[@]}"; do + [[ "${NODE_IP[$node]}" != "$VIP_IP" ]] \ + || die "VIP ${VIP_IP} is already the static management address of ${node}." + done + + # Sort nodes by descending priority. This matters with nopreempt: start the + # highest-priority node first so it deterministically becomes the initial MASTER. + mapfile -t SORTED_NODES < <( + for node in "${NODES[@]}"; do + printf '%s %s\n' "${NODE_PRIORITY[$node]}" "$node" + done | sort -nr | awk '{print $2}' + ) + + printf '\nDeployment plan:\n' + printf ' VIP: %s\n' "$VIP_CIDR" + printf ' Interface: %s\n' "$INTERFACE" + printf ' VRID: %s\n' "$VRID" + printf ' Failback: disabled (nopreempt)\n\n' + printf ' %-28s %-16s %-8s\n' "NODE" "MANAGEMENT IP" "PRIORITY" + printf ' %-28s %-16s %-8s\n' "----------------------------" "----------------" "--------" + for node in "${SORTED_NODES[@]}"; do + printf ' %-28s %-16s %-8s\n' "$node" "${NODE_IP[$node]}" "${NODE_PRIORITY[$node]}" + done + + if (( ASSUME_YES == 0 )); then + printf '\nThis will install/reconfigure Keepalived on every listed node.\n' + read -r -p 'Proceed? [y/N] ' answer + [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } + fi + + HEALTH_FILE="$TMPDIR/check-proxmox-keepalived.sh" + cat > "$HEALTH_FILE" <<'HEALTH' +#!/usr/bin/env bash +set -u + +# A node may own the management VIP only if the Proxmox management stack, +# Corosync and cluster quorum are all healthy. + +systemctl is-active --quiet pveproxy || exit 1 +systemctl is-active --quiet pvedaemon || exit 1 +systemctl is-active --quiet pve-cluster || exit 1 +systemctl is-active --quiet corosync || exit 1 + +# A surviving but non-quorate node is intentionally ineligible for the VIP. +timeout 3 pvecm status 2>/dev/null \ + | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$' || exit 1 + +# Verify the actual GUI/API endpoint is answering locally. +curl --silent --show-error --fail --insecure --max-time 3 \ + https://127.0.0.1:8006/ \ + >/dev/null 2>&1 || exit 1 + +exit 0 +HEALTH + chmod 0750 "$HEALTH_FILE" + + log "Installing Keepalived/curl and staging configuration on all nodes." + + for node in "${NODES[@]}"; do + log "Preparing ${node}" + + remote_exec "$node" \ + "DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y keepalived curl >/dev/null" + + remote_health="/tmp/check-proxmox-keepalived.${DEPLOY_TAG}" + copy_to_node "$node" "$HEALTH_FILE" "$remote_health" + remote_exec "$node" \ + "install -o root -g root -m 0750 '$remote_health' /usr/local/sbin/check-proxmox-keepalived.sh && rm -f '$remote_health'" + + config_file="$TMPDIR/keepalived-${node}.conf" + + { + cat < "$config_file" + + remote_conf="/tmp/keepalived.conf.${DEPLOY_TAG}" + copy_to_node "$node" "$config_file" "$remote_conf" + + # Validate before replacing the live config. + remote_exec "$node" "keepalived -t -f '$remote_conf'" \ + || die "Keepalived rejected the generated config for ${node}. Existing config was not replaced." + + remote_exec "$node" \ + "if [[ -f /etc/keepalived/keepalived.conf ]]; then cp -a /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.pre-proxmox-vip.${DEPLOY_TAG}; fi; install -o root -g root -m 0644 '$remote_conf' /etc/keepalived/keepalived.conf; rm -f '$remote_conf'; systemctl enable keepalived >/dev/null" + + # Confirm the health script itself is currently passing. + remote_exec "$node" "/usr/local/sbin/check-proxmox-keepalived.sh" \ + || die "Health check failed on ${node}; Keepalived has not been restarted yet." + done + + log "All configurations validated. Resetting Keepalived election state." + + # Stop all instances so a rerun cannot preserve an unintended existing master. + for node in "${NODES[@]}"; do + remote_exec "$node" "systemctl stop keepalived" || true + done + + MASTER_NODE="${SORTED_NODES[0]}" + MASTER_IP="${NODE_IP[$MASTER_NODE]}" + + log "Starting highest-priority node first: ${MASTER_NODE} (${NODE_PRIORITY[$MASTER_NODE]})." + remote_exec "$MASTER_NODE" "systemctl start keepalived" + + # Wait until the preferred initial node actually owns the VIP before starting + # nopreempt backups. This prevents a lower-priority node from winning first. + master_ready=0 + for _ in {1..12}; do + if remote_exec "$MASTER_NODE" \ + "ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then + master_ready=1 + break + fi + sleep 1 + done + + (( master_ready == 1 )) \ + || die "${MASTER_NODE} did not acquire VIP ${VIP_IP}. Keepalived remains stopped on the other nodes. Check: journalctl -u keepalived" + + for node in "${SORTED_NODES[@]:1}"; do + log "Starting backup node ${node}." + remote_exec "$node" "systemctl start keepalived" + done + + log "Verifying VIP ownership and service state." + + owners=() + for node in "${NODES[@]}"; do + remote_exec "$node" "systemctl is-active --quiet keepalived" \ + || die "Keepalived is not active on ${node}." + + if remote_exec "$node" \ + "ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then + owners+=("$node") + fi + done + + (( ${#owners[@]} == 1 )) \ + || die "Expected exactly one VIP owner, found ${#owners[@]}: ${owners[*]:-none}" + + # Verify the VIP itself answers the Proxmox API from the deployment node. + if ! curl --silent --show-error --fail --insecure --max-time 5 \ + "https://${VIP_IP}:8006/" >/dev/null 2>&1; then + warn "VIP ${VIP_IP} is assigned to ${owners[0]}, but the API test through the VIP failed." + warn "Check routing/firewall rules and whether pveproxy has been restricted to a specific LISTEN_IP." + else + log "VIP API test succeeded." + fi + + cat <