Update proxmenu-scripts.sh
This commit is contained in:
+243
-240
@@ -284,39 +284,39 @@ INSTALL_KEEPALIVE() {
|
|||||||
* )
|
* )
|
||||||
echo
|
echo
|
||||||
|
|
||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------
|
||||||
# Proxmox cluster Keepalived deployment
|
# Proxmox cluster Keepalived deployment
|
||||||
#
|
#
|
||||||
# - Discovers cluster members with `pvecm nodes`
|
# - Discovers cluster members with `pvecm nodes`
|
||||||
# - Uses the first non-VIP IPv4 address on the selected interface as each
|
# - Uses the first non-VIP IPv4 address on the selected interface as each
|
||||||
# node's Keepalived unicast source address
|
# node's Keepalived unicast source address
|
||||||
# - Derives VRRP priority from the hostname's trailing numeric suffix
|
# - Derives VRRP priority from the hostname's trailing numeric suffix
|
||||||
# priority = PRIORITY_BASE - trailing_number
|
# priority = PRIORITY_BASE - trailing_number
|
||||||
# so pve1 > pve2 > pve3 when PRIORITY_BASE=200
|
# so pve1 > pve2 > pve3 when PRIORITY_BASE=200
|
||||||
# - Deploys a Proxmox-aware health check including cluster quorum
|
# - Deploys a Proxmox-aware health check including cluster quorum
|
||||||
# - Uses nopreempt so the VIP does not fail back after a recovered node returns
|
# - Uses nopreempt so the VIP does not fail back after a recovered node returns
|
||||||
# - Starts the highest-priority node first so initial ownership is deterministic
|
# - Starts the highest-priority node first so initial ownership is deterministic
|
||||||
#
|
#
|
||||||
# Run this from any healthy, quorate Proxmox cluster node as root.
|
# Run this from any healthy, quorate Proxmox cluster node as root.
|
||||||
# -----------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
VIP_CIDR="10.2.1.10/24"
|
VIP_CIDR="10.2.1.10/24"
|
||||||
INTERFACE="vmbr0"
|
INTERFACE="vmbr0"
|
||||||
PRIORITY_BASE=200
|
PRIORITY_BASE=200
|
||||||
VRID=51
|
VRID=51
|
||||||
ADVERT_INT=1
|
ADVERT_INT=1
|
||||||
ASSUME_YES=0
|
ASSUME_YES=0
|
||||||
|
|
||||||
SSH_OPTS=(
|
SSH_OPTS=(
|
||||||
-o BatchMode=yes
|
-o BatchMode=yes
|
||||||
-o ConnectTimeout=8
|
-o ConnectTimeout=8
|
||||||
-o StrictHostKeyChecking=accept-new
|
-o StrictHostKeyChecking=accept-new
|
||||||
)
|
)
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
cat <<USAGE
|
cat <<USAGE
|
||||||
Usage: $0 [options]
|
Usage: $0 [options]
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
@@ -332,46 +332,46 @@ Examples:
|
|||||||
$0 --vip 10.2.1.10/24 --interface vmbr0
|
$0 --vip 10.2.1.10/24 --interface vmbr0
|
||||||
$0 --vip 192.168.50.10/24 --priority-base 220 --vrid 61 -y
|
$0 --vip 192.168.50.10/24 --priority-base 220 --vrid 61 -y
|
||||||
USAGE
|
USAGE
|
||||||
}
|
}
|
||||||
|
|
||||||
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
|
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
|
||||||
warn() { printf '\nWARNING: %s\n' "$*" >&2; }
|
warn() { printf '\nWARNING: %s\n' "$*" >&2; }
|
||||||
die() { printf '\nERROR: %s\n' "$*" >&2; exit 1; }
|
die() { printf '\nERROR: %s\n' "$*" >&2; exit 1; }
|
||||||
|
|
||||||
while [[ $# -gt 0 ]]; do
|
while [[ $# -gt 0 ]]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--vip)
|
--vip)
|
||||||
[[ $# -ge 2 ]] || die "--vip requires a value"
|
[[ $# -ge 2 ]] || die "--vip requires a value"
|
||||||
VIP_CIDR="$2"; shift 2 ;;
|
VIP_CIDR="$2"; shift 2 ;;
|
||||||
--interface)
|
--interface)
|
||||||
[[ $# -ge 2 ]] || die "--interface requires a value"
|
[[ $# -ge 2 ]] || die "--interface requires a value"
|
||||||
INTERFACE="$2"; shift 2 ;;
|
INTERFACE="$2"; shift 2 ;;
|
||||||
--priority-base)
|
--priority-base)
|
||||||
[[ $# -ge 2 ]] || die "--priority-base requires a value"
|
[[ $# -ge 2 ]] || die "--priority-base requires a value"
|
||||||
PRIORITY_BASE="$2"; shift 2 ;;
|
PRIORITY_BASE="$2"; shift 2 ;;
|
||||||
--vrid)
|
--vrid)
|
||||||
[[ $# -ge 2 ]] || die "--vrid requires a value"
|
[[ $# -ge 2 ]] || die "--vrid requires a value"
|
||||||
VRID="$2"; shift 2 ;;
|
VRID="$2"; shift 2 ;;
|
||||||
-y|--yes)
|
-y|--yes)
|
||||||
ASSUME_YES=1; shift ;;
|
ASSUME_YES=1; shift ;;
|
||||||
-h|--help)
|
-h|--help)
|
||||||
usage; exit 0 ;;
|
usage; exit 0 ;;
|
||||||
*)
|
*)
|
||||||
die "Unknown option: $1" ;;
|
die "Unknown option: $1" ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
[[ $EUID -eq 0 ]] || die "Run this script as root."
|
[[ $EUID -eq 0 ]] || die "Run this script as root."
|
||||||
command -v pvecm >/dev/null 2>&1 || die "pvecm not found. Run this on a Proxmox VE node."
|
command -v pvecm >/dev/null 2>&1 || die "pvecm not found. Run this on a Proxmox VE node."
|
||||||
command -v python3 >/dev/null 2>&1 || die "python3 is required."
|
command -v python3 >/dev/null 2>&1 || die "python3 is required."
|
||||||
|
|
||||||
[[ "$INTERFACE" =~ ^[A-Za-z0-9_.:-]+$ ]] || die "Invalid interface name: $INTERFACE"
|
[[ "$INTERFACE" =~ ^[A-Za-z0-9_.:-]+$ ]] || die "Invalid interface name: $INTERFACE"
|
||||||
[[ "$PRIORITY_BASE" =~ ^[0-9]+$ ]] || die "Priority base must be numeric."
|
[[ "$PRIORITY_BASE" =~ ^[0-9]+$ ]] || die "Priority base must be numeric."
|
||||||
[[ "$VRID" =~ ^[0-9]+$ ]] || die "VRID must be numeric."
|
[[ "$VRID" =~ ^[0-9]+$ ]] || die "VRID must be numeric."
|
||||||
(( VRID >= 1 && VRID <= 255 )) || die "VRID must be between 1 and 255."
|
(( VRID >= 1 && VRID <= 255 )) || die "VRID must be between 1 and 255."
|
||||||
|
|
||||||
# Validate VIP/CIDR and derive the bare IP.
|
# Validate VIP/CIDR and derive the bare IP.
|
||||||
VIP_IP="$(python3 - "$VIP_CIDR" <<'PY'
|
VIP_IP="$(python3 - "$VIP_CIDR" <<'PY'
|
||||||
import ipaddress, sys
|
import ipaddress, sys
|
||||||
try:
|
try:
|
||||||
iface = ipaddress.ip_interface(sys.argv[1])
|
iface = ipaddress.ip_interface(sys.argv[1])
|
||||||
@@ -383,135 +383,135 @@ if iface.version != 4:
|
|||||||
raise SystemExit(1)
|
raise SystemExit(1)
|
||||||
print(iface.ip)
|
print(iface.ip)
|
||||||
PY
|
PY
|
||||||
)" || die "Invalid VIP/CIDR: $VIP_CIDR"
|
)" || die "Invalid VIP/CIDR: $VIP_CIDR"
|
||||||
|
|
||||||
# This check intentionally uses the Proxmox-reported quorum state.
|
# This check intentionally uses the Proxmox-reported quorum state.
|
||||||
if ! timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'; then
|
if ! timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'; then
|
||||||
die "The local node does not currently see a quorate cluster. Refusing deployment."
|
die "The local node does not currently see a quorate cluster. Refusing deployment."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
LOCAL_NODE="$(pvecm nodes | awk '$NF == "(local)" {print $3; exit}')"
|
LOCAL_NODE="$(pvecm nodes | awk '$NF == "(local)" {print $3; exit}')"
|
||||||
[[ -n "$LOCAL_NODE" ]] || LOCAL_NODE="$(hostname -s)"
|
[[ -n "$LOCAL_NODE" ]] || LOCAL_NODE="$(hostname -s)"
|
||||||
DEPLOY_TAG="$(date +%Y%m%d-%H%M%S)-$$"
|
DEPLOY_TAG="$(date +%Y%m%d-%H%M%S)-$$"
|
||||||
TMPDIR="$(mktemp -d)"
|
TMPDIR="$(mktemp -d)"
|
||||||
trap 'rm -rf "$TMPDIR"' EXIT
|
trap 'rm -rf "$TMPDIR"' EXIT
|
||||||
|
|
||||||
remote_exec() {
|
remote_exec() {
|
||||||
local node="$1"
|
local node="$1"
|
||||||
local cmd="$2"
|
local cmd="$2"
|
||||||
|
|
||||||
if [[ "$node" == "$LOCAL_NODE" ]]; then
|
if [[ "$node" == "$LOCAL_NODE" ]]; then
|
||||||
bash -lc "$cmd"
|
bash -lc "$cmd"
|
||||||
else
|
else
|
||||||
ssh "${SSH_OPTS[@]}" "root@${node}" "$cmd"
|
ssh "${SSH_OPTS[@]}" "root@${node}" "$cmd"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
copy_to_node() {
|
copy_to_node() {
|
||||||
local node="$1"
|
local node="$1"
|
||||||
local src="$2"
|
local src="$2"
|
||||||
local dst="$3"
|
local dst="$3"
|
||||||
|
|
||||||
if [[ "$node" == "$LOCAL_NODE" ]]; then
|
if [[ "$node" == "$LOCAL_NODE" ]]; then
|
||||||
cp -f "$src" "$dst"
|
cp -f "$src" "$dst"
|
||||||
else
|
else
|
||||||
scp -q "${SSH_OPTS[@]}" "$src" "root@${node}:${dst}"
|
scp -q "${SSH_OPTS[@]}" "$src" "root@${node}:${dst}"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Discover cluster node names. pvecm nodes emits: Nodeid Votes Name [(local)]
|
# Discover cluster node names. pvecm nodes emits: Nodeid Votes Name [(local)]
|
||||||
mapfile -t NODES < <(pvecm nodes | awk '$1 ~ /^(0x)?[0-9A-Fa-f]+$/ && $2 ~ /^[0-9]+$/ {print $3}')
|
mapfile -t NODES < <(pvecm nodes | awk '$1 ~ /^(0x)?[0-9A-Fa-f]+$/ && $2 ~ /^[0-9]+$/ {print $3}')
|
||||||
(( ${#NODES[@]} >= 2 )) || die "Fewer than two cluster nodes were discovered."
|
(( ${#NODES[@]} >= 2 )) || die "Fewer than two cluster nodes were discovered."
|
||||||
|
|
||||||
declare -A NODE_IP
|
declare -A NODE_IP
|
||||||
declare -A NODE_PRIORITY
|
declare -A NODE_PRIORITY
|
||||||
declare -A PRIORITY_OWNER
|
declare -A PRIORITY_OWNER
|
||||||
|
|
||||||
log "Preflight: discovered ${#NODES[@]} cluster nodes."
|
log "Preflight: discovered ${#NODES[@]} cluster nodes."
|
||||||
|
|
||||||
# First pass: verify SSH/access, interface addresses, health, suffixes and priorities.
|
# First pass: verify SSH/access, interface addresses, health, suffixes and priorities.
|
||||||
for node in "${NODES[@]}"; do
|
for node in "${NODES[@]}"; do
|
||||||
if [[ "$node" != "$LOCAL_NODE" ]]; then
|
if [[ "$node" != "$LOCAL_NODE" ]]; then
|
||||||
ssh "${SSH_OPTS[@]}" "root@${node}" true \
|
ssh "${SSH_OPTS[@]}" "root@${node}" true \
|
||||||
|| die "Passwordless root SSH to ${node} failed. No changes have been made."
|
|| die "Passwordless root SSH to ${node} failed. No changes have been made."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
remote_exec "$node" "ip link show '$INTERFACE' >/dev/null 2>&1" \
|
remote_exec "$node" "ip link show '$INTERFACE' >/dev/null 2>&1" \
|
||||||
|| die "Interface ${INTERFACE} does not exist on ${node}."
|
|| die "Interface ${INTERFACE} does not exist on ${node}."
|
||||||
|
|
||||||
# Exclude the VIP so the script remains safe/idempotent if Keepalived is
|
# Exclude the VIP so the script remains safe/idempotent if Keepalived is
|
||||||
# already running and this deployment is being refreshed.
|
# already running and this deployment is being refreshed.
|
||||||
ip="$(remote_exec "$node" \
|
ip="$(remote_exec "$node" \
|
||||||
"ip -4 -o addr show dev '$INTERFACE' scope global | awk '{split(\$4,a,\"/\"); print a[1]}' | grep -vx '$VIP_IP' | head -n1")"
|
"ip -4 -o addr show dev '$INTERFACE' scope global | awk '{split(\$4,a,\"/\"); print a[1]}' | grep -vx '$VIP_IP' | head -n1")"
|
||||||
|
|
||||||
[[ -n "$ip" ]] || die "Could not determine a non-VIP IPv4 address on ${node}:${INTERFACE}."
|
[[ -n "$ip" ]] || die "Could not determine a non-VIP IPv4 address on ${node}:${INTERFACE}."
|
||||||
[[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || die "Unexpected IPv4 address '${ip}' on ${node}."
|
[[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || die "Unexpected IPv4 address '${ip}' on ${node}."
|
||||||
|
|
||||||
# Verify each management address resides inside the VIP's subnet.
|
# Verify each management address resides inside the VIP's subnet.
|
||||||
python3 - "$VIP_CIDR" "$ip" <<'PY' >/dev/null || die "${node} (${ip}) is not in the VIP subnet ${VIP_CIDR}."
|
python3 - "$VIP_CIDR" "$ip" <<'PY' >/dev/null || die "${node} (${ip}) is not in the VIP subnet ${VIP_CIDR}."
|
||||||
import ipaddress, sys
|
import ipaddress, sys
|
||||||
vip = ipaddress.ip_interface(sys.argv[1])
|
vip = ipaddress.ip_interface(sys.argv[1])
|
||||||
ip = ipaddress.ip_address(sys.argv[2])
|
ip = ipaddress.ip_address(sys.argv[2])
|
||||||
raise SystemExit(0 if ip in vip.network else 1)
|
raise SystemExit(0 if ip in vip.network else 1)
|
||||||
PY
|
PY
|
||||||
|
|
||||||
[[ "$node" =~ ([0-9]+)$ ]] \
|
[[ "$node" =~ ([0-9]+)$ ]] \
|
||||||
|| die "Hostname '${node}' has no trailing numeric suffix."
|
|| die "Hostname '${node}' has no trailing numeric suffix."
|
||||||
|
|
||||||
# 10# prevents numbers with leading zeroes from being interpreted as octal.
|
# 10# prevents numbers with leading zeroes from being interpreted as octal.
|
||||||
suffix=$((10#${BASH_REMATCH[1]}))
|
suffix=$((10#${BASH_REMATCH[1]}))
|
||||||
priority=$((PRIORITY_BASE - suffix))
|
priority=$((PRIORITY_BASE - suffix))
|
||||||
|
|
||||||
(( priority >= 1 && priority <= 254 )) \
|
(( priority >= 1 && priority <= 254 )) \
|
||||||
|| die "Calculated priority ${priority} for ${node} is outside 1..254. Adjust --priority-base."
|
|| die "Calculated priority ${priority} for ${node} is outside 1..254. Adjust --priority-base."
|
||||||
|
|
||||||
if [[ -n "${PRIORITY_OWNER[$priority]:-}" ]]; then
|
if [[ -n "${PRIORITY_OWNER[$priority]:-}" ]]; then
|
||||||
die "Priority collision: ${node} and ${PRIORITY_OWNER[$priority]} both calculate to ${priority}."
|
die "Priority collision: ${node} and ${PRIORITY_OWNER[$priority]} both calculate to ${priority}."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Verify the remote node itself currently sees quorum before touching it.
|
# Verify the remote node itself currently sees quorum before touching it.
|
||||||
remote_exec "$node" \
|
remote_exec "$node" \
|
||||||
"timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'" \
|
"timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'" \
|
||||||
|| die "${node} does not currently report Quorate: Yes. No changes have been made."
|
|| die "${node} does not currently report Quorate: Yes. No changes have been made."
|
||||||
|
|
||||||
NODE_IP["$node"]="$ip"
|
NODE_IP["$node"]="$ip"
|
||||||
NODE_PRIORITY["$node"]="$priority"
|
NODE_PRIORITY["$node"]="$priority"
|
||||||
PRIORITY_OWNER["$priority"]="$node"
|
PRIORITY_OWNER["$priority"]="$node"
|
||||||
done
|
done
|
||||||
|
|
||||||
# Prevent accidentally assigning a VIP equal to a real node address.
|
# Prevent accidentally assigning a VIP equal to a real node address.
|
||||||
for node in "${NODES[@]}"; do
|
for node in "${NODES[@]}"; do
|
||||||
[[ "${NODE_IP[$node]}" != "$VIP_IP" ]] \
|
[[ "${NODE_IP[$node]}" != "$VIP_IP" ]] \
|
||||||
|| die "VIP ${VIP_IP} is already the static management address of ${node}."
|
|| die "VIP ${VIP_IP} is already the static management address of ${node}."
|
||||||
done
|
done
|
||||||
|
|
||||||
# Sort nodes by descending priority. This matters with nopreempt: start the
|
# Sort nodes by descending priority. This matters with nopreempt: start the
|
||||||
# highest-priority node first so it deterministically becomes the initial MASTER.
|
# highest-priority node first so it deterministically becomes the initial MASTER.
|
||||||
mapfile -t SORTED_NODES < <(
|
mapfile -t SORTED_NODES < <(
|
||||||
for node in "${NODES[@]}"; do
|
for node in "${NODES[@]}"; do
|
||||||
printf '%s %s\n' "${NODE_PRIORITY[$node]}" "$node"
|
printf '%s %s\n' "${NODE_PRIORITY[$node]}" "$node"
|
||||||
done | sort -nr | awk '{print $2}'
|
done | sort -nr | awk '{print $2}'
|
||||||
)
|
)
|
||||||
|
|
||||||
printf '\nDeployment plan:\n'
|
printf '\nDeployment plan:\n'
|
||||||
printf ' VIP: %s\n' "$VIP_CIDR"
|
printf ' VIP: %s\n' "$VIP_CIDR"
|
||||||
printf ' Interface: %s\n' "$INTERFACE"
|
printf ' Interface: %s\n' "$INTERFACE"
|
||||||
printf ' VRID: %s\n' "$VRID"
|
printf ' VRID: %s\n' "$VRID"
|
||||||
printf ' Failback: disabled (nopreempt)\n\n'
|
printf ' Failback: disabled (nopreempt)\n\n'
|
||||||
printf ' %-28s %-16s %-8s\n' "NODE" "MANAGEMENT IP" "PRIORITY"
|
printf ' %-28s %-16s %-8s\n' "NODE" "MANAGEMENT IP" "PRIORITY"
|
||||||
printf ' %-28s %-16s %-8s\n' "----------------------------" "----------------" "--------"
|
printf ' %-28s %-16s %-8s\n' "----------------------------" "----------------" "--------"
|
||||||
for node in "${SORTED_NODES[@]}"; do
|
for node in "${SORTED_NODES[@]}"; do
|
||||||
printf ' %-28s %-16s %-8s\n' "$node" "${NODE_IP[$node]}" "${NODE_PRIORITY[$node]}"
|
printf ' %-28s %-16s %-8s\n' "$node" "${NODE_IP[$node]}" "${NODE_PRIORITY[$node]}"
|
||||||
done
|
done
|
||||||
|
|
||||||
if (( ASSUME_YES == 0 )); then
|
if (( ASSUME_YES == 0 )); then
|
||||||
printf '\nThis will install/reconfigure Keepalived on every listed node.\n'
|
printf '\nThis will install/reconfigure Keepalived on every listed node.\n'
|
||||||
read -r -p 'Proceed? [y/N] ' answer
|
read -r -p 'Proceed? [y/N] ' answer
|
||||||
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
|
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
|
||||||
fi
|
fi
|
||||||
|
|
||||||
HEALTH_FILE="$TMPDIR/check-proxmox-keepalived.sh"
|
HEALTH_FILE="$TMPDIR/check-proxmox-keepalived.sh"
|
||||||
cat > "$HEALTH_FILE" <<'HEALTH'
|
cat > "$HEALTH_FILE" <<'HEALTH'
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
set -u
|
set -u
|
||||||
|
|
||||||
@@ -534,25 +534,25 @@ curl --silent --show-error --fail --insecure --max-time 3 \
|
|||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
HEALTH
|
HEALTH
|
||||||
chmod 0750 "$HEALTH_FILE"
|
chmod 0750 "$HEALTH_FILE"
|
||||||
|
|
||||||
log "Installing Keepalived/curl and staging configuration on all nodes."
|
log "Installing Keepalived/curl and staging configuration on all nodes."
|
||||||
|
|
||||||
for node in "${NODES[@]}"; do
|
for node in "${NODES[@]}"; do
|
||||||
log "Preparing ${node}"
|
log "Preparing ${node}"
|
||||||
|
|
||||||
remote_exec "$node" \
|
remote_exec "$node" \
|
||||||
"DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y keepalived curl >/dev/null"
|
"DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y keepalived curl >/dev/null"
|
||||||
|
|
||||||
remote_health="/tmp/check-proxmox-keepalived.${DEPLOY_TAG}"
|
remote_health="/tmp/check-proxmox-keepalived.${DEPLOY_TAG}"
|
||||||
copy_to_node "$node" "$HEALTH_FILE" "$remote_health"
|
copy_to_node "$node" "$HEALTH_FILE" "$remote_health"
|
||||||
remote_exec "$node" \
|
remote_exec "$node" \
|
||||||
"install -o root -g root -m 0750 '$remote_health' /usr/local/sbin/check-proxmox-keepalived.sh && rm -f '$remote_health'"
|
"install -o root -g root -m 0750 '$remote_health' /usr/local/sbin/check-proxmox-keepalived.sh && rm -f '$remote_health'"
|
||||||
|
|
||||||
config_file="$TMPDIR/keepalived-${node}.conf"
|
config_file="$TMPDIR/keepalived-${node}.conf"
|
||||||
|
|
||||||
{
|
{
|
||||||
cat <<CONF
|
cat <<CONF
|
||||||
# Managed by deploy-proxmox-keepalived.sh
|
# Managed by deploy-proxmox-keepalived.sh
|
||||||
# Deployment: ${DEPLOY_TAG}
|
# Deployment: ${DEPLOY_TAG}
|
||||||
|
|
||||||
@@ -583,12 +583,12 @@ vrrp_instance PROXMOX_VIP {
|
|||||||
unicast_peer {
|
unicast_peer {
|
||||||
CONF
|
CONF
|
||||||
|
|
||||||
for peer in "${NODES[@]}"; do
|
for peer in "${NODES[@]}"; do
|
||||||
[[ "$peer" == "$node" ]] && continue
|
[[ "$peer" == "$node" ]] && continue
|
||||||
printf ' %s\n' "${NODE_IP[$peer]}"
|
printf ' %s\n' "${NODE_IP[$peer]}"
|
||||||
done
|
done
|
||||||
|
|
||||||
cat <<CONF
|
cat <<CONF
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual_ipaddress {
|
virtual_ipaddress {
|
||||||
@@ -600,82 +600,82 @@ CONF
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
CONF
|
CONF
|
||||||
} > "$config_file"
|
} > "$config_file"
|
||||||
|
|
||||||
remote_conf="/tmp/keepalived.conf.${DEPLOY_TAG}"
|
remote_conf="/tmp/keepalived.conf.${DEPLOY_TAG}"
|
||||||
copy_to_node "$node" "$config_file" "$remote_conf"
|
copy_to_node "$node" "$config_file" "$remote_conf"
|
||||||
|
|
||||||
# Validate before replacing the live config.
|
# Validate before replacing the live config.
|
||||||
remote_exec "$node" "keepalived -t -f '$remote_conf'" \
|
remote_exec "$node" "keepalived -t -f '$remote_conf'" \
|
||||||
|| die "Keepalived rejected the generated config for ${node}. Existing config was not replaced."
|
|| die "Keepalived rejected the generated config for ${node}. Existing config was not replaced."
|
||||||
|
|
||||||
remote_exec "$node" \
|
remote_exec "$node" \
|
||||||
"if [[ -f /etc/keepalived/keepalived.conf ]]; then cp -a /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.pre-proxmox-vip.${DEPLOY_TAG}; fi; install -o root -g root -m 0644 '$remote_conf' /etc/keepalived/keepalived.conf; rm -f '$remote_conf'; systemctl enable keepalived >/dev/null"
|
"if [[ -f /etc/keepalived/keepalived.conf ]]; then cp -a /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.pre-proxmox-vip.${DEPLOY_TAG}; fi; install -o root -g root -m 0644 '$remote_conf' /etc/keepalived/keepalived.conf; rm -f '$remote_conf'; systemctl enable keepalived >/dev/null"
|
||||||
|
|
||||||
# Confirm the health script itself is currently passing.
|
# Confirm the health script itself is currently passing.
|
||||||
remote_exec "$node" "/usr/local/sbin/check-proxmox-keepalived.sh" \
|
remote_exec "$node" "/usr/local/sbin/check-proxmox-keepalived.sh" \
|
||||||
|| die "Health check failed on ${node}; Keepalived has not been restarted yet."
|
|| die "Health check failed on ${node}; Keepalived has not been restarted yet."
|
||||||
done
|
done
|
||||||
|
|
||||||
log "All configurations validated. Resetting Keepalived election state."
|
log "All configurations validated. Resetting Keepalived election state."
|
||||||
|
|
||||||
# Stop all instances so a rerun cannot preserve an unintended existing master.
|
# Stop all instances so a rerun cannot preserve an unintended existing master.
|
||||||
for node in "${NODES[@]}"; do
|
for node in "${NODES[@]}"; do
|
||||||
remote_exec "$node" "systemctl stop keepalived" || true
|
remote_exec "$node" "systemctl stop keepalived" || true
|
||||||
done
|
done
|
||||||
|
|
||||||
MASTER_NODE="${SORTED_NODES[0]}"
|
MASTER_NODE="${SORTED_NODES[0]}"
|
||||||
MASTER_IP="${NODE_IP[$MASTER_NODE]}"
|
MASTER_IP="${NODE_IP[$MASTER_NODE]}"
|
||||||
|
|
||||||
log "Starting highest-priority node first: ${MASTER_NODE} (${NODE_PRIORITY[$MASTER_NODE]})."
|
log "Starting highest-priority node first: ${MASTER_NODE} (${NODE_PRIORITY[$MASTER_NODE]})."
|
||||||
remote_exec "$MASTER_NODE" "systemctl start keepalived"
|
remote_exec "$MASTER_NODE" "systemctl start keepalived"
|
||||||
|
|
||||||
# Wait until the preferred initial node actually owns the VIP before starting
|
# Wait until the preferred initial node actually owns the VIP before starting
|
||||||
# nopreempt backups. This prevents a lower-priority node from winning first.
|
# nopreempt backups. This prevents a lower-priority node from winning first.
|
||||||
master_ready=0
|
master_ready=0
|
||||||
for _ in {1..12}; do
|
for _ in {1..12}; do
|
||||||
if remote_exec "$MASTER_NODE" \
|
if remote_exec "$MASTER_NODE" \
|
||||||
"ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then
|
"ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then
|
||||||
master_ready=1
|
master_ready=1
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
|
||||||
(( master_ready == 1 )) \
|
(( master_ready == 1 )) \
|
||||||
|| die "${MASTER_NODE} did not acquire VIP ${VIP_IP}. Keepalived remains stopped on the other nodes. Check: journalctl -u keepalived"
|
|| die "${MASTER_NODE} did not acquire VIP ${VIP_IP}. Keepalived remains stopped on the other nodes. Check: journalctl -u keepalived"
|
||||||
|
|
||||||
for node in "${SORTED_NODES[@]:1}"; do
|
for node in "${SORTED_NODES[@]:1}"; do
|
||||||
log "Starting backup node ${node}."
|
log "Starting backup node ${node}."
|
||||||
remote_exec "$node" "systemctl start keepalived"
|
remote_exec "$node" "systemctl start keepalived"
|
||||||
done
|
done
|
||||||
|
|
||||||
log "Verifying VIP ownership and service state."
|
log "Verifying VIP ownership and service state."
|
||||||
|
|
||||||
owners=()
|
owners=()
|
||||||
for node in "${NODES[@]}"; do
|
for node in "${NODES[@]}"; do
|
||||||
remote_exec "$node" "systemctl is-active --quiet keepalived" \
|
remote_exec "$node" "systemctl is-active --quiet keepalived" \
|
||||||
|| die "Keepalived is not active on ${node}."
|
|| die "Keepalived is not active on ${node}."
|
||||||
|
|
||||||
if remote_exec "$node" \
|
if remote_exec "$node" \
|
||||||
"ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then
|
"ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then
|
||||||
owners+=("$node")
|
owners+=("$node")
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
(( ${#owners[@]} == 1 )) \
|
(( ${#owners[@]} == 1 )) \
|
||||||
|| die "Expected exactly one VIP owner, found ${#owners[@]}: ${owners[*]:-none}"
|
|| die "Expected exactly one VIP owner, found ${#owners[@]}: ${owners[*]:-none}"
|
||||||
|
|
||||||
# Verify the VIP itself answers the Proxmox API from the deployment node.
|
# Verify the VIP itself answers the Proxmox API from the deployment node.
|
||||||
if ! curl --silent --show-error --fail --insecure --max-time 5 \
|
if ! curl --silent --show-error --fail --insecure --max-time 5 \
|
||||||
"https://${VIP_IP}:8006/" >/dev/null 2>&1; then
|
"https://${VIP_IP}:8006/" >/dev/null 2>&1; then
|
||||||
warn "VIP ${VIP_IP} is assigned to ${owners[0]}, but the API test through the VIP failed."
|
warn "VIP ${VIP_IP} is assigned to ${owners[0]}, but the API test through the VIP failed."
|
||||||
warn "Check routing/firewall rules and whether pveproxy has been restricted to a specific LISTEN_IP."
|
warn "Check routing/firewall rules and whether pveproxy has been restricted to a specific LISTEN_IP."
|
||||||
else
|
else
|
||||||
log "VIP API test succeeded."
|
log "VIP API test succeeded."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
cat <<RESULT
|
cat <<RESULT
|
||||||
|
|
||||||
Deployment complete.
|
Deployment complete.
|
||||||
|
|
||||||
@@ -702,6 +702,9 @@ Useful commands:
|
|||||||
/usr/local/sbin/check-proxmox-keepalived.sh; echo \$?
|
/usr/local/sbin/check-proxmox-keepalived.sh; echo \$?
|
||||||
RESULT
|
RESULT
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
echo -e "\n${idsCL[Green]}Keepalive has been installed${idsCL[Default]}"
|
echo -e "\n${idsCL[Green]}Keepalive has been installed${idsCL[Default]}"
|
||||||
[ ${action-x} ] && exit 0 || ENTER2CONTINUE
|
[ ${action-x} ] && exit 0 || ENTER2CONTINUE
|
||||||
esac
|
esac
|
||||||
|
|||||||
Reference in New Issue
Block a user