This commit is contained in:
2025-05-31 17:46:12 -05:00
parent a0fd6bc834
commit 8719d80068
3 changed files with 235 additions and 212 deletions

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
PW_VERS='2.10.1h-05312025'
PW_VERS='2.10.1m-05312025'
noheader=' update service dailytemp confsync -r -report test '

View File

@@ -838,7 +838,7 @@ BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'V ~ (`IDS_NUMBER_FORMAT ${BATTLIFE
fi
if [ ! -f ${PW_TMPFOLDER}/power.ac.good ]; then
if [ -f ${PW_TMPFOLDER}/shutdown.SYS ] || [ -f ${PW_TMPFOLDER}/shutdown.MAIN ] || [ -f ${PW_TMPFOLDER}/power.ac.conserve ]; then
if [[ "${IP}" != "10.6.1"* ]] && ([ -f ${PW_TMPFOLDER}/shutdown.SYS ] || [ -f ${PW_TMPFOLDER}/shutdown.MAIN ] || [ -f ${PW_TMPFOLDER}/power.ac.conserve ]); then
SENDNOTICE "HOSTS POWER" "Power restored! Will wait ${PW_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on"
touch ${PW_TMPFOLDER}/.power.restored
fi

View File

@@ -11,7 +11,7 @@ source /opt/idssys/powerwall/inc/power.inc.sh
source /opt/idssys/powerwall/inc/temp.inc.sh
# TEST=true
[ "${TESTMODE_ENABLE}" == "true" ] && TEST=true
shopt -s lastpipe
@@ -239,105 +239,16 @@ SHUTDOWN_SERVER(){
hostname=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}
fi
echo
DIVIDER . yellow ${PRI_WIDTH}
[ ${MM} -eq 1 ] && smsg="Entering Maintenance Mode" || smsg="Shutting down"
echo -e "${idsCL[LightYellow]}${hostname}${idsCL[Yellow]} - ${smsg}${idsCL[Default]}"
DIVIDER . yellow ${PRI_WIDTH}
echo
if [ "${ESXI_HOSTS}" != "" ]; then
if [ ${MM} -eq 1 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${hostname} into maintenance mode" >> ${PW_LOGFILE}
else
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${hostname}" >> ${PW_LOGFILE}
fi
##INCREASE CPU OVERCOMMIT
if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${ESXI_HOST}" != "OFFSITE" ]; then
MSG="Increasing DRS CPU-Overcommit to 4"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/set-drs.ps1 4 >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n"
fi
fi
if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then
if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
MSG="Stopping all iSCSI-PRI VM's"
elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then
MSG="Stopping all iSCSI-SYS VM's"
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Stopping all iSCSI VM's"
fi
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 >/dev/null 2>&1 &
fi
sleep 5s
echo -e "${idsCL[Green]}Command sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
if [ ${MM} -eq 0 ] && [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then
# if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ]; then
# MSG="Disabling CLS for the Offsite Cluster in vCenter"
# c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
# echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
# if [ "${TEST}" != "true" ]; then
# run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
# touch ${PW_TMPFOLDER}/shutdown.cls.offsite
# else
# echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
# fi
# fi
if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ]); then
MSG="Disabling CLS for the Primary Cluster in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
touch ${PW_TMPFOLDER}/shutdown.cls.mainsite
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
MSG="Restarting vCenter CLS Services"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
if [[ "${IP}" == "10.6.1"* ]]; then
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Enabling maintenance mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
MSG="Shutting down: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
[[ "${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]}" = "10.6."* ]] && ESXI_PASS=${PW_ESXI_PASS_KP} || ESXI_PASS=${PW_ESXI_PASS}
if [ "${TEST}" != "true" ]; then
if [ "${TEST^^}" != "TRUE" ] && [ "${TEST_DONOT_SHUTDOWN^^}" != "TRUE" ]; then
if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
sleep 10s
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} "/vmfs/volumes/ESXi-Vault (Fileserver)/!SCRIPTS/esxidown/async.sh"
echo -e "${idsCL[Green]}Command sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}"
@@ -346,108 +257,110 @@ SHUTDOWN_SERVER(){
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
else
echo
if [ ${MM} -eq 0 ] && ([[ "${ESXI_HOST}" == *"10.6.1."* ]] || [ "${ESXI_HOST}" == "ALL" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then
tmp=tmp
else
MSG="Starting iSCSI migrations"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
sleep 80s # up from 45s, then 60...
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
DIVIDER . yellow ${PRI_WIDTH}
[ ${MM} -eq 1 ] && smsg="Entering Maintenance Mode" || smsg="Shutting down"
echo -e "${idsCL[LightYellow]}${hostname}${idsCL[Yellow]} - ${smsg}${idsCL[Default]}"
DIVIDER . yellow ${PRI_WIDTH}
echo
if [ "${ESXI_HOSTS}" != "" ]; then
if [ ${MM} -eq 1 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${hostname} into maintenance mode" >> ${PW_LOGFILE}
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${hostname}" >> ${PW_LOGFILE}
fi
echo
fi
if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then
MSG="Waiting for iSCSI VM's to power down"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: "
# for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
##INCREASE CPU OVERCOMMIT
if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${ESXI_HOST}" != "OFFSITE" ]; then
MSG="Increasing DRS CPU-Overcommit to 4"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${1}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${1}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 >/dev/null 2>&1
elif [ "${1}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${1}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 >/dev/null 2>&1
elif [ "${1}" == "MAINSITE" ] || [ "${1}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/set-drs.ps1 4 >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n"
fi
fi
if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then
if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
MSG="Stopping all iSCSI-PRI VM's"
elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then
MSG="Stopping all iSCSI-SYS VM's"
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Stopping all iSCSI VM's"
fi
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 >/dev/null 2>&1 &
fi
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
# if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then
# VERIFY_ISCSI_VM_DOWN "${ESXI_HOST}" &
# else
# touch ${PW_TMPFOLDER}/verify.iscsi.${ESXIHOST}.done
# fi
sleep 5s
echo -e "${idsCL[Green]}Command sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
# done
# if [ "${TEST}" != "true" ]; then
# nc_count=0
# until [ -f ${PW_TMPFOLDER}/verify.iscsi.done ]; do
# sleep 1s
# done
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
# else
# echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
# fi
if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
MSG="Shutting down iSCSI-PRI Server"
elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then
MSG="Shutting down iSCSI-SYS Server"
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Shutting down both iSCSI-PRI & iSCSI-SYS Servers"
echo
fi
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 &
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 &
if [ ${MM} -eq 0 ] && [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then
# if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ]; then
# MSG="Disabling CLS for the Offsite Cluster in vCenter"
# c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
# echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
# if [ "${TEST}" != "true" ]; then
# run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
# touch ${PW_TMPFOLDER}/shutdown.cls.offsite
# else
# echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
# fi
# fi
if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ]); then
MSG="Disabling CLS for the Primary Cluster in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
touch ${PW_TMPFOLDER}/shutdown.cls.mainsite
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
echo -e "${idsCL[Green]}Command sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
MSG="Restarting vCenter CLS Services"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
echo
fi
MSG="Shutting down any remaining VM's"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
varvms="${PW_VCENTER_HOST_NAMES[${ESXIHOST}]},${varvms}"
done
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 "${varvms::-1}" >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Command sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
if [ ${MM} -eq 0 ]; then
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
MSG="Enabling maintenance mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
[[ "${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]}" = "10.6."* ]] && ESXI_PASS=${PW_ESXI_PASS_KP} || ESXI_PASS=${PW_ESXI_PASS}
if [ "${TEST^^}" != "TRUE" ] && [ "${TEST_DONOT_SHUTDOWN^^}" != "TRUE" ]; then
if [ "${TEST}" != "true" ]; then
if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}"
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
sleep 10s
echo -e "${idsCL[Green]}Command sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}"
fi
@@ -457,52 +370,162 @@ SHUTDOWN_SERVER(){
done
echo
if [ "${ESXI_HOST}" == "${PW_MAIN_ISCSI}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
MSG="Waiting for ${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]} to shutdown"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}/${idsCL[LightCyan]}${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}${idsCL[White]}}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "${PW_MAIN_ISCSI}">/dev/null 2>&1; do sleep 1; done
sleep 5s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
if [ ${MM} -eq 0 ] && ([[ "${ESXI_HOST}" == *"10.6.1."* ]] || [ "${ESXI_HOST}" == "ALL" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then
tmp=tmp
MSG="Powering off SAS Enclosure"
else
MSG="Starting iSCSI migrations"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST ${PW_MAIN_ISCSI})" == "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2)
if [ "${TEST}" != "true" ]; then
sleep 80s # up from 45s, then 60...
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
[ "${ESXI_HOST}" != "${PW_MAIN_ISCSI}" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n"
touch ${PW_TMPFOLDER}/shutdown.${ESXI_HOST}
touch ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}.disable
else
echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be in maintenance mode momentarily ... \n"
touch ${PW_TMPFOLDER}/maintmode.${ESXI_HOST}
fi
## DECREASE CPU OVERCOMMIT
if [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${HOST^^}" != "OFFSITE" ]; then
MSG="Decreasing DRS CPU-Overcommit back down to 2"
if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then
MSG="Waiting for iSCSI VM's to power down"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: "
# for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
if [ "${TEST}" != "true" ]; then
if [ "${1}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${1}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 >/dev/null 2>&1
elif [ "${1}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${1}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 >/dev/null 2>&1
elif [ "${1}" == "MAINSITE" ] || [ "${1}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 >/dev/null 2>&1
fi
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
# if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then
# VERIFY_ISCSI_VM_DOWN "${ESXI_HOST}" &
# else
# touch ${PW_TMPFOLDER}/verify.iscsi.${ESXIHOST}.done
# fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
# done
# if [ "${TEST}" != "true" ]; then
# nc_count=0
# until [ -f ${PW_TMPFOLDER}/verify.iscsi.done ]; do
# sleep 1s
# done
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
# else
# echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
# fi
if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
MSG="Shutting down iSCSI-PRI Server"
elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then
MSG="Shutting down iSCSI-SYS Server"
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Shutting down both iSCSI-PRI & iSCSI-SYS Servers"
fi
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 &
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 &
fi
echo -e "${idsCL[Green]}Command sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
MSG="Shutting down any remaining VM's"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
{ sleep 60s; ${PW_FOLDER}/esxi-scripts/set-drs.ps1 2 >/dev/null 2>&1; } &
echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n"
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
varvms="${PW_VCENTER_HOST_NAMES[${ESXIHOST}]},${varvms}"
done
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 "${varvms::-1}" >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Command sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n"
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
echo
else
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
if [ ${MM} -eq 0 ]; then
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
[[ "${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]}" = "10.6."* ]] && ESXI_PASS=${PW_ESXI_PASS_KP} || ESXI_PASS=${PW_ESXI_PASS}
if [ "${TEST^^}" != "TRUE" ] && [ "${TEST_DONOT_SHUTDOWN^^}" != "TRUE" ]; then
if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${ESXI_HOST}" == "${PW_MAIN_ISCSI}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then
MSG="Waiting for ${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]} to shutdown"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}/${idsCL[LightCyan]}${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}${idsCL[White]}}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "${PW_MAIN_ISCSI}">/dev/null 2>&1; do sleep 1; done
sleep 5s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
MSG="Powering off SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST ${PW_MAIN_ISCSI})" == "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
[ "${ESXI_HOST}" != "${PW_MAIN_ISCSI}" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n"
touch ${PW_TMPFOLDER}/shutdown.${ESXI_HOST}
touch ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}.disable
else
echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be in maintenance mode momentarily ... \n"
touch ${PW_TMPFOLDER}/maintmode.${ESXI_HOST}
fi
## DECREASE CPU OVERCOMMIT
if [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${HOST^^}" != "OFFSITE" ]; then
MSG="Decreasing DRS CPU-Overcommit back down to 2"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
{ sleep 60s; ${PW_FOLDER}/esxi-scripts/set-drs.ps1 2 >/dev/null 2>&1; } &
echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n"
fi
fi
else
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
fi
fi
echo
}