diff --git a/defaults.inc b/defaults.inc index 183c56f3..f0bfd715 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ -VERS='2.5.5-12112023' +VERS='2.5.6-12112023' noheader=' update service dailytemp confsync ' hc_cw=60 diff --git a/esxi-scripts/esxi-shutdown-inmain.sh b/esxi-scripts/esxi-shutdown-inmain.sh index ad5abf20..6acc5b09 100644 --- a/esxi-scripts/esxi-shutdown-inmain.sh +++ b/esxi-scripts/esxi-shutdown-inmain.sh @@ -74,7 +74,7 @@ fi # shutdown the ESXi host echo "Shutting down ESXi host after 10 seconds..." if [ $TEST -eq 0 ]; then - esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown - esxidown.sh" + esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown" fi # exit the session diff --git a/esxi-scripts/esxi-shutdown.sh b/esxi-scripts/esxi-shutdown.sh index dfb49bd7..232809b5 100755 --- a/esxi-scripts/esxi-shutdown.sh +++ b/esxi-scripts/esxi-shutdown.sh @@ -71,7 +71,7 @@ echo "Guest VM shutdown complete..." # shutdown the ESXi host echo "Shutting down ESXi host after 10 seconds..." if [ $TEST -eq 0 ]; then - esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown - esxidown.sh" + esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown" fi # exit maintenance mode immediately before server has a chance to shutdown/power off diff --git a/powerwall.sh b/powerwall.sh index 7cef26fe..37ef7d8d 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -7,6 +7,7 @@ source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc source /opt/idssys/powerwall/defaults.inc source ${PW_FOLDER}/settings.conf +source ${PW_FOLDER}/shutdown-test.sh TEST=true @@ -1773,7 +1774,7 @@ MAINT_MODE_VERIFY_SHUTDOWN(){ sleep 3s MaintMode=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get') done - ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' + ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"' ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' if [ "${ESXIHOST}" == "10.10.2.10" ]; then @@ -2346,7 +2347,9 @@ fi [ "${2^^}" == "SYS" ] && ISCSISYSVMSHUTDOWN || ISCSIVMSHUTDOWN ;; shutdownhost) SHUTDOWN_SERVER ${2} ${3};; + shutdowntest) SHUTDOWN_SERVER2 ${2} ${3};; shutdown_servers) SHUTDOWN_SERVERS ${2} ${3};; + hostpower) HOSTPOWER ${2} ${3};; host) HOSTMGMT ${2} ${3} ${4} ${5} ${6} ${7};; diff --git a/shutdown-test.sh b/shutdown-test.sh new file mode 100644 index 00000000..3f2c32ff --- /dev/null +++ b/shutdown-test.sh @@ -0,0 +1,136 @@ +SHUTDOWN_SERVER2(){ + if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then + tmp=${PW_iDRACHOST_SHORTNAMES[${1}]} + ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]} + elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then + ESXIHOST=${1} + elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then + ESXIHOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${1}]} + elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then + tmp=${PW_HOST_IDRACS[${1}]} + ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]} + else + ESXIHOST='' + fi + + + if [ "${ESXIHOST}" != "" ]; then + host=${PW_ESXI_HOST_NAMES[${ESXIHOST}]} + if [ "${2}" == "maintenance" ]; then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${host} [${ESXIHOST}] into maintenance mode" >> ${PW_LOGFILE} + else + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${host} [${ESXIHOST}]" >> ${PW_LOGFILE} + fi + + if [ "${ESXIHOST}" == "10.10.2.10" ]; then + echo -e "${idsCL[LightYellow]}Since this is the iSCSI-PRI host server, all iSCSI-PRI VM's are being shutdown ... " + for ehost in "${PW_MAINSITEHOSTS[@]}"; do + MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]" + c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 & + echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" + sleep 5s + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + done + echo + [ "${TEST}" != "true" ] && sleep 10s + + elif [ "${ESXIHOST}" == "10.10.2.17" ]; then + echo -e "${idsCL[LightYellow]}Since this is the iSCSI-SYS host server, all iSCSI-SYS VM's are being shutdown ... " + for ehost in "${PW_MAINSITEHOSTS[@]}"; do + MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]" + c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 & + echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" + sleep 5s + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + done + echo + [ "${TEST}" != "true" ] && sleep 10s + + fi + + MSG="Enabling maintenance mode & starting iSCSI VM migrations" + c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 & + sleep 25s + echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + + MSG="Shutting down remaining VM's" + c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) + if [ "${ESXIHOST}" == "10.10.2.10" ] || [ "${ESXIHOST}" == "10.10.2.17" ]; then + if [ "${ESXIHOST}" == "10.10.2.10" ]; then + run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) + elif [ "${ESXIHOST}" == "10.10.2.17" ]; then + run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) + fi + run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) + fi + run=$(${PW_FOLDER}/esxi-scripts/cls-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) + echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + + if [ "${2}" != "maintenance" ]; then + MSG="Waiting for VM's to power off or migrate" + c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & + echo -e "${idsCL[Green]}Continuing in background\n" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + + + if [ "${ESXIHOST}" == "10.10.2.10" ]; then + MSG="Waiting for Host ${host} [${ESXIHOST}] to shutdown" + c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done + echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + + MSG="Powering off SAS Enclosure" + c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then + run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2) + sleep 3s + echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + + else + echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be shutdown momentarily ... \n" + fi + else + echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be in maintenance mode momentarily ... \n" + fi + + else + echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}" + fi + echo +} \ No newline at end of file