diff --git a/defaults.inc b/defaults.inc index e35b55b6..5386e81d 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -PW_VERS='2.10.1h-05312025' +PW_VERS='2.10.1m-05312025' noheader=' update service dailytemp confsync -r -report test ' diff --git a/inc/power.inc.sh b/inc/power.inc.sh index abde2a2e..6dad8907 100644 --- a/inc/power.inc.sh +++ b/inc/power.inc.sh @@ -838,7 +838,7 @@ BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'V ~ (`IDS_NUMBER_FORMAT ${BATTLIFE fi if [ ! -f ${PW_TMPFOLDER}/power.ac.good ]; then - if [ -f ${PW_TMPFOLDER}/shutdown.SYS ] || [ -f ${PW_TMPFOLDER}/shutdown.MAIN ] || [ -f ${PW_TMPFOLDER}/power.ac.conserve ]; then + if [[ "${IP}" != "10.6.1"* ]] && ([ -f ${PW_TMPFOLDER}/shutdown.SYS ] || [ -f ${PW_TMPFOLDER}/shutdown.MAIN ] || [ -f ${PW_TMPFOLDER}/power.ac.conserve ]); then SENDNOTICE "HOSTS POWER" "Power restored! Will wait ${PW_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on" touch ${PW_TMPFOLDER}/.power.restored fi diff --git a/powerwall.sh b/powerwall.sh index 713e62bb..75e418c0 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -11,7 +11,7 @@ source /opt/idssys/powerwall/inc/power.inc.sh source /opt/idssys/powerwall/inc/temp.inc.sh -# TEST=true +[ "${TESTMODE_ENABLE}" == "true" ] && TEST=true shopt -s lastpipe @@ -239,105 +239,16 @@ SHUTDOWN_SERVER(){ hostname=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]} fi - echo - DIVIDER . yellow ${PRI_WIDTH} - [ ${MM} -eq 1 ] && smsg="Entering Maintenance Mode" || smsg="Shutting down" - echo -e "${idsCL[LightYellow]}${hostname}${idsCL[Yellow]} - ${smsg}${idsCL[Default]}" - DIVIDER . yellow ${PRI_WIDTH} - echo - - if [ "${ESXI_HOSTS}" != "" ]; then - if [ ${MM} -eq 1 ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${hostname} into maintenance mode" >> ${PW_LOGFILE} - else - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${hostname}" >> ${PW_LOGFILE} - fi - - ##INCREASE CPU OVERCOMMIT - if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${ESXI_HOST}" != "OFFSITE" ]; then - MSG="Increasing DRS CPU-Overcommit to 4" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/set-drs.ps1 4 >/dev/null 2>&1 & - echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n" - fi - fi - - if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then - if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then - MSG="Stopping all iSCSI-PRI VM's" - elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then - MSG="Stopping all iSCSI-SYS VM's" - elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then - MSG="Stopping all iSCSI VM's" - fi - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 >/dev/null 2>&1 & - elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 >/dev/null 2>&1 & - elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 >/dev/null 2>&1 & - fi - sleep 5s - echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi - - if [ ${MM} -eq 0 ] && [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then - # if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ]; then - # MSG="Disabling CLS for the Offsite Cluster in vCenter" - # c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - # echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - # if [ "${TEST}" != "true" ]; then - # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - # echo -e "${idsCL[Green]}Done${idsCL[Default]}" - # touch ${PW_TMPFOLDER}/shutdown.cls.offsite - # else - # echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - # fi - # fi - if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ]); then - MSG="Disabling CLS for the Primary Cluster in vCenter" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - touch ${PW_TMPFOLDER}/shutdown.cls.mainsite - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - fi - MSG="Restarting vCenter CLS Services" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 & - echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi + if [[ "${IP}" == "10.6.1"* ]]; then for ESXIHOST in "${ESXI_HOSTS[@]}"; do - MSG="Enabling maintenance mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + MSG="Shutting down: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " [[ "${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]}" = "10.6."* ]] && ESXI_PASS=${PW_ESXI_PASS_KP} || ESXI_PASS=${PW_ESXI_PASS} - if [ "${TEST}" != "true" ]; then + if [ "${TEST^^}" != "TRUE" ] && [ "${TEST_DONOT_SHUTDOWN^^}" != "TRUE" ]; then if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then - ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 & - sleep 10s + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} "/vmfs/volumes/ESXi-Vault (Fileserver)/!SCRIPTS/esxidown/async.sh" echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" else echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" @@ -346,108 +257,110 @@ SHUTDOWN_SERVER(){ echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi done + + + else + echo - - if [ ${MM} -eq 0 ] && ([[ "${ESXI_HOST}" == *"10.6.1."* ]] || [ "${ESXI_HOST}" == "ALL" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then - tmp=tmp - - else - MSG="Starting iSCSI migrations" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - sleep 80s # up from 45s, then 60... - echo -e "${idsCL[Green]}Done${idsCL[Default]}" + DIVIDER . yellow ${PRI_WIDTH} + [ ${MM} -eq 1 ] && smsg="Entering Maintenance Mode" || smsg="Shutting down" + echo -e "${idsCL[LightYellow]}${hostname}${idsCL[Yellow]} - ${smsg}${idsCL[Default]}" + DIVIDER . yellow ${PRI_WIDTH} + echo + + if [ "${ESXI_HOSTS}" != "" ]; then + if [ ${MM} -eq 1 ]; then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${hostname} into maintenance mode" >> ${PW_LOGFILE} else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${hostname}" >> ${PW_LOGFILE} fi - echo - fi - if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then - MSG="Waiting for iSCSI VM's to power down" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: " - # for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do + ##INCREASE CPU OVERCOMMIT + if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${ESXI_HOST}" != "OFFSITE" ]; then + MSG="Increasing DRS CPU-Overcommit to 4" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - if [ "${1}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${1}" == "MAIN" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 >/dev/null 2>&1 - elif [ "${1}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${1}" == "SYS" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 >/dev/null 2>&1 - elif [ "${1}" == "MAINSITE" ] || [ "${1}" == "ALL" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 >/dev/null 2>&1 + ${PW_FOLDER}/esxi-scripts/set-drs.ps1 4 >/dev/null 2>&1 & + echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n" + fi + fi + + if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then + if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then + MSG="Stopping all iSCSI-PRI VM's" + elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then + MSG="Stopping all iSCSI-SYS VM's" + elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + MSG="Stopping all iSCSI VM's" + fi + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 >/dev/null 2>&1 & fi - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - # if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then - # VERIFY_ISCSI_VM_DOWN "${ESXI_HOST}" & - # else - # touch ${PW_TMPFOLDER}/verify.iscsi.${ESXIHOST}.done - # fi + sleep 5s + echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi - # done - - # if [ "${TEST}" != "true" ]; then - # nc_count=0 - # until [ -f ${PW_TMPFOLDER}/verify.iscsi.done ]; do - # sleep 1s - # done - # echo -e "${idsCL[Green]}Done${idsCL[Default]}" - # else - # echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - # fi - - if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then - MSG="Shutting down iSCSI-PRI Server" - elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then - MSG="Shutting down iSCSI-SYS Server" - elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then - MSG="Shutting down both iSCSI-PRI & iSCSI-SYS Servers" + echo fi - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 & - elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 & - elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 & - ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 & + + if [ ${MM} -eq 0 ] && [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then + # if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ]; then + # MSG="Disabling CLS for the Offsite Cluster in vCenter" + # c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + # echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + # if [ "${TEST}" != "true" ]; then + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + # echo -e "${idsCL[Green]}Done${idsCL[Default]}" + # touch ${PW_TMPFOLDER}/shutdown.cls.offsite + # else + # echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + # fi + # fi + if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ]); then + MSG="Disabling CLS for the Primary Cluster in vCenter" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + touch ${PW_TMPFOLDER}/shutdown.cls.mainsite + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi fi - echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + MSG="Restarting vCenter CLS Services" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 & + echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + echo fi - echo - fi - MSG="Shutting down any remaining VM's" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then for ESXIHOST in "${ESXI_HOSTS[@]}"; do - varvms="${PW_VCENTER_HOST_NAMES[${ESXIHOST}]},${varvms}" - done - ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 "${varvms::-1}" >/dev/null 2>&1 & - echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" - - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - - if [ ${MM} -eq 0 ]; then - for ESXIHOST in "${ESXI_HOSTS[@]}"; do - MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + MSG="Enabling maintenance mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " [[ "${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]}" = "10.6."* ]] && ESXI_PASS=${PW_ESXI_PASS_KP} || ESXI_PASS=${PW_ESXI_PASS} - if [ "${TEST^^}" != "TRUE" ] && [ "${TEST_DONOT_SHUTDOWN^^}" != "TRUE" ]; then + if [ "${TEST}" != "true" ]; then if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then - MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & - echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" + ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 & + sleep 10s + echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" else echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" fi @@ -457,52 +370,162 @@ SHUTDOWN_SERVER(){ done echo - if [ "${ESXI_HOST}" == "${PW_MAIN_ISCSI}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then - MSG="Waiting for ${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]} to shutdown" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG//${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}/${idsCL[LightCyan]}${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}${idsCL[White]}}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - while ping -qw 10 -c3 "${PW_MAIN_ISCSI}">/dev/null 2>&1; do sleep 1; done - sleep 5s - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi + if [ ${MM} -eq 0 ] && ([[ "${ESXI_HOST}" == *"10.6.1."* ]] || [ "${ESXI_HOST}" == "ALL" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then + tmp=tmp - MSG="Powering off SAS Enclosure" + else + MSG="Starting iSCSI migrations" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST ${PW_MAIN_ISCSI})" == "false" ]; then - run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2) + if [ "${TEST}" != "true" ]; then + sleep 80s # up from 45s, then 60... echo -e "${idsCL[Green]}Done${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi - + echo fi - [ "${ESXI_HOST}" != "${PW_MAIN_ISCSI}" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n" - touch ${PW_TMPFOLDER}/shutdown.${ESXI_HOST} - touch ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}.disable - else - echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be in maintenance mode momentarily ... \n" - touch ${PW_TMPFOLDER}/maintmode.${ESXI_HOST} - fi - ## DECREASE CPU OVERCOMMIT - if [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${HOST^^}" != "OFFSITE" ]; then - MSG="Decreasing DRS CPU-Overcommit back down to 2" + if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then + MSG="Waiting for iSCSI VM's to power down" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: " + # for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do + if [ "${TEST}" != "true" ]; then + if [ "${1}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${1}" == "MAIN" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 >/dev/null 2>&1 + elif [ "${1}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${1}" == "SYS" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 >/dev/null 2>&1 + elif [ "${1}" == "MAINSITE" ] || [ "${1}" == "ALL" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 >/dev/null 2>&1 + fi + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + # if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then + # VERIFY_ISCSI_VM_DOWN "${ESXI_HOST}" & + # else + # touch ${PW_TMPFOLDER}/verify.iscsi.${ESXIHOST}.done + # fi + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + # done + + # if [ "${TEST}" != "true" ]; then + # nc_count=0 + # until [ -f ${PW_TMPFOLDER}/verify.iscsi.done ]; do + # sleep 1s + # done + # echo -e "${idsCL[Green]}Done${idsCL[Default]}" + # else + # echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + # fi + + if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then + MSG="Shutting down iSCSI-PRI Server" + elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then + MSG="Shutting down iSCSI-SYS Server" + elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + MSG="Shutting down both iSCSI-PRI & iSCSI-SYS Servers" + fi + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 & + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 & + fi + echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + echo + fi + + MSG="Shutting down any remaining VM's" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - { sleep 60s; ${PW_FOLDER}/esxi-scripts/set-drs.ps1 2 >/dev/null 2>&1; } & - echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n" + for ESXIHOST in "${ESXI_HOSTS[@]}"; do + varvms="${PW_VCENTER_HOST_NAMES[${ESXIHOST}]},${varvms}" + done + ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 "${varvms::-1}" >/dev/null 2>&1 & + echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" + else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n" + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi - fi + echo - else - echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}" + if [ ${MM} -eq 0 ]; then + for ESXIHOST in "${ESXI_HOSTS[@]}"; do + MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " + [[ "${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]}" = "10.6."* ]] && ESXI_PASS=${PW_ESXI_PASS_KP} || ESXI_PASS=${PW_ESXI_PASS} + if [ "${TEST^^}" != "TRUE" ] && [ "${TEST_DONOT_SHUTDOWN^^}" != "TRUE" ]; then + if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then + MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & + echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" + fi + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + done + echo + + if [ "${ESXI_HOST}" == "${PW_MAIN_ISCSI}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then + MSG="Waiting for ${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]} to shutdown" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG//${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}/${idsCL[LightCyan]}${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}${idsCL[White]}}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + while ping -qw 10 -c3 "${PW_MAIN_ISCSI}">/dev/null 2>&1; do sleep 1; done + sleep 5s + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + + MSG="Powering off SAS Enclosure" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST ${PW_MAIN_ISCSI})" == "false" ]; then + run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2) + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + + fi + [ "${ESXI_HOST}" != "${PW_MAIN_ISCSI}" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n" + touch ${PW_TMPFOLDER}/shutdown.${ESXI_HOST} + touch ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}.disable + else + echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be in maintenance mode momentarily ... \n" + touch ${PW_TMPFOLDER}/maintmode.${ESXI_HOST} + fi + + ## DECREASE CPU OVERCOMMIT + if [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${HOST^^}" != "OFFSITE" ]; then + MSG="Decreasing DRS CPU-Overcommit back down to 2" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + { sleep 60s; ${PW_FOLDER}/esxi-scripts/set-drs.ps1 2 >/dev/null 2>&1; } & + echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n" + fi + fi + + else + echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}" + fi fi echo }