From 40e49f60bf6dfb0ceb1bb5c4de0e84bc389f0310 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Wed, 13 Dec 2023 21:33:21 -0600 Subject: [PATCH] update --- powerwall.sh | 828 ++++++++++++++--------------------------------- shutdown-test.sh | 2 +- 2 files changed, 239 insertions(+), 591 deletions(-) diff --git a/powerwall.sh b/powerwall.sh index f46c7a4c..d682f484 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -923,14 +923,14 @@ BATT VOLTAGE: ${BATTVOLT}'volts" 1 touch ${PW_TMPFOLDER}/power.ac.low ##### SHUTDOWN OFFSITE SERVER WHILE ITS AT MAINSITE - SHUTDOWN_SERVERS2 10.2.1.11 & + SHUTDOWN_SERVER 10.2.1.11 & fi if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_CONSERVE_AFTER}*60" | bc) ]; then echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGPOWER} SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1 - SHUTDOWN_SERVERS2 ${PW_HOST_IDRACS['iDS-vMS-Host0']} & + SHUTDOWN_SERVER ${PW_HOST_IDRACS['iDS-vMS-Host0']} & touch ${PW_TMPFOLDER}/power.ac.conserve fi fi @@ -1215,15 +1215,15 @@ CHECK_SERVICES(){ SHUTDOWN_MAIN(){ if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "TEMP" ]; then - # SHUTDOWN_SERVERS MAIN ${1} - SHUTDOWN_SERVER2 main + # SHUTDOWN_SERVER MAIN ${1} + SHUTDOWN_SERVER main # HOSTMGMT main -e off >/dev/null 2>&1 reason='Server Room Overheated!' sendnotice=true elif [ "${1}" == "POWER" ]; then - # SHUTDOWN_SERVERS MAIN ${1} - SHUTDOWN_SERVER2 main + # SHUTDOWN_SERVER MAIN ${1} + SHUTDOWN_SERVER main # HOSTMGMT main -e off >/dev/null 2>&1 reason='Low Power!' sendnotice=true @@ -1242,15 +1242,15 @@ ${reason}" 1 } SHUTDOWN_SYS(){ if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "TEMP" ]; then - # SHUTDOWN_SERVERS SYS ${1} - [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ] && SHUTDOWN_SERVER2 mainsite || SHUTDOWN_SERVER2 sys + # SHUTDOWN_SERVER SYS ${1} + [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ] && SHUTDOWN_SERVER mainsite || SHUTDOWN_SERVER sys # HOSTMGMT sys -e off >/dev/null 2>&1 reason='Server Room Overheated!' sendnotice=true elif [ "${1}" == "POWER" ]; then - # SHUTDOWN_SERVERS SYS ${1} - [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ] && SHUTDOWN_SERVER2 mainsite || SHUTDOWN_SERVER2 sys + # SHUTDOWN_SERVER SYS ${1} + [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ] && SHUTDOWN_SERVER mainsite || SHUTDOWN_SERVER sys # HOSTMGMT sys -e off >/dev/null 2>&1 reason='Low Power!' sendnotice=true @@ -1291,490 +1291,16 @@ SHUTDOWN_CRIT(){ ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/reboot fi } + SHUTDOWN_OFFSITE(){ - if [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then + if [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down offsite host(s)" 1 for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do SHUTDOWN_SERVER ${offsite_host} & done - touch ${PW_TMPFOLDER}/power.offsite.off fi } -SHUTDOWN_SERVERS(){ - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${1} servers" >> ${PW_LOGFILE} - [ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE} - - if [ "${1^^}" == "MAIN" ]; then - if [ ! -f ${PW_TMPFOLDER}/power.main.off ]; then - touch ${PW_TMPFOLDER}/power.main.off - - if [ "${2}" != "maintenance" ]; then - MSG="Disabling CLS in vCenter" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi - - for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - MSG="Verifying iSCSI-PRI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - if [ "${2}" != "maintenance" ]; then - for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & - echo -e "${idsCL[Green]}Continuing in background\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - MSG="Waiting for iSCSI-PRI Host to shutdown" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - MSG="Powering OFF the SAS Enclosure" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then - run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1) - sleep 3s - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n" - fi - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi - fi - - elif [ "${1^^}" == "SYS" ]; then - [ ! -f ${PW_TMPFOLDER}/power.main.off ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN TEMP - if [ ! -f ${PW_TMPFOLDER}/power.sys.off ]; then - touch ${PW_TMPFOLDER}/power.sys.off - - if [ "${2}" != "maintenance" ]; then - MSG="Disabling CLS in vCenter" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi - - for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - MSG="Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 - ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - if [ "${2}" != "maintenance" ]; then - for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & - echo -e "${idsCL[Green]}Continuing in background\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - fi - fi - - elif [ "${1^^}" == "ALL" ]; then - if [ ! -f ${PW_TMPFOLDER}/power.all.off ]; then - touch ${PW_TMPFOLDER}/power.all.off - - if [ "${2}" != "maintenance" ]; then - MSG="Disabling CLS in vCenter" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi - - for ESXIHOST in "${PW_ALLHOSTS[@]}"; do - MSG="Stopping all iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1 - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_ALLHOSTS[@]}"; do - MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_ALLHOSTS[@]}"; do - MSG="Verifying iSCSI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_ALLHOSTS[@]}"; do - MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1 - ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1 - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - if [ "${2}" != "maintenance" ]; then - for ESXIHOST in "${PW_ALLHOSTS[@]}"; do - MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & - echo -e "${idsCL[Green]}Continuing in background\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - MSG="Waiting for iSCSI-PRI Host to shutdown" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - MSG="Powering OFF the SAS Enclosure" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then - run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1) - sleep 3s - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n" - fi - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi - fi - - elif [ "${1^^}" == "MAINSITE" ]; then - if [ ! -f ${PW_TMPFOLDER}/power.main.offsite ]; then - touch ${PW_TMPFOLDER}/power.main.offsite - - if [ "${2}" != "maintenance" ]; then - MSG="Disabling CLS in vCenter" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi - - for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do - MSG="Stopping iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do - MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do - MSG="Verifying iSCSI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do - MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - if [ "${2}" != "maintenance" ]; then - for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do - MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & - echo -e "${idsCL[Green]}Continuing in background\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - MSG="Waiting for iSCSI-PRI Host to shutdown" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - MSG="Powering OFF the SAS Enclosure" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then - run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1) - sleep 3s - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n" - fi - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi - fi - - elif [ "${1^^}" == "OFFSITE" ]; then - if [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then - touch ${PW_TMPFOLDER}/power.offsite.off - - if [ "${2}" != "maintenance" ]; then - MSG="Disabling CLS in vCenter" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - echo - fi - - for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do - MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do - MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - if [ "${2}" != "maintenance" ]; then - for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do - MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & - echo -e "${idsCL[Green]}Continuing in background\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - fi - fi - fi -} MAINT_MODE_VERIFY_SHUTDOWN(){ ESXIHOST=${1} MaintMode="" @@ -1782,139 +1308,261 @@ MAINT_MODE_VERIFY_SHUTDOWN(){ sleep 3s MaintMode=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get') done - # ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"' - # ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' + ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"' + ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' } + SHUTDOWN_SERVER(){ - if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then - tmp=${PW_iDRACHOST_SHORTNAMES[${1}]} - ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]} - elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then - ESXIHOST=${1} - elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then - ESXIHOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${1}]} - elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then - tmp=${PW_HOST_IDRACS[${1}]} - ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]} - else - ESXIHOST='' + if [ "${1}" != "-h" ]; then + HOST=${1}; MN=0 + if [ "${PW_iDRACHOST_SHORTNAMES[${HOST}]}" != "" ]; then + tmp=${PW_iDRACHOST_SHORTNAMES[${HOST}]} + ESXI_HOST=${PW_HOST_ADDRESSES[${tmp}]} + elif [ "${PW_ESXI_HOST_NAMES[${HOST}]}" != "" ]; then + ESXI_HOST=${HOST} + elif [ "${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]}" != "" ]; then + ESXI_HOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]} + elif ([ "${HOST::1}" == "o" ] || [ "${HOST::1}" == "O" ]) && [ "${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]}" != "" ]; then + ESXI_HOST=${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]} + elif [ "${PW_HOST_ADDRESSES[${HOST}]}" != "" ]; then + ESXI_HOST=${PW_HOST_ADDRESSES[${HOST}]} + elif [ "${1^^}" == "ALL" ] || [ "${1^^}" == "MAINSITE" ] || [ "${1^^}" == "OFFSITE" ] || [ "${1^^}" == "MAIN" ] || [ "${1^^}" == "SYS" ]; then + ESXI_HOST=${HOST^^} + MN=1 + else + ESXI_HOST='' + fi + [ "${ESXI_HOST}" == "" ] && echo -e "Invalid host\n" && exit 1 + if [ "${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}" != "" ]; then + tmp=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]} + idracip=${PW_HOST_IDRACS[${tmp}]} + hostname=${PW_iDRACHOST_SHORTNAMES[${idracip}]} + fi fi + while [ $# -gt 0 ]; do + case "$1" in + -p|-power) POWER=${2}; MODE="power";; + -e|-esxi) ESXI=${2}; MODE="esxi";; + -h) + echo -e "Usage: ${idsCL[Yellow]}pw host {host} {flags}${idsCL[Default]} {" + width=35 + printf "%-${width}s- %s\n" " {host}" "(*required, can either be esxiIP,idracIP or Host#)" + printf "%-${width}s- %s\n" " -p|-power {on / off}" "Controls Host Power" + printf "%-${width}s- %s\n" " -e|-esxi {off / mm}" "Shuts down ESXi or puts into maintenance mode" + echo -e "}\n" + exit 0 + ;; + esac + shift 1 + done - if [ "${ESXIHOST}" != "" ]; then - host=${PW_ESXI_HOST_NAMES[${ESXIHOST}]} + if [ ${MN} -eq 1 ]; then + ESXI_HOSTS=PW_${ESXI_HOST}HOSTS[@] + ESXI_HOSTS=(${!ESXI_HOSTS}) + [ "${ESXI_HOST}" == "ALL" ] && hostname="All Host Servers" || hostname="All ${ESXI_HOST} Host Servers" + else + ESXI_HOSTS=(${ESXI_HOST}) + hostname=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]} + fi + + echo + DIVIDER . yellow 75 + echo -e "${idsCL[Yellow]}Shutting down - ${hostname}${idsCL[Default]}" + DIVIDER . yellow 75 + + if [ "${ESXI_HOSTS}" != "" ]; then if [ "${2}" == "maintenance" ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${host} [${ESXIHOST}] into maintenance mode" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${hostname} into maintenance mode" >> ${PW_LOGFILE} else - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${host} [${ESXIHOST}]" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${hostname}" >> ${PW_LOGFILE} fi - if [ "${ESXIHOST}" == "10.10.2.10" ]; then - echo -e "${idsCL[LightYellow]}Since this is the iSCSI-PRI host server, all iSCSI-PRI VM's are being shutdown ... " - for ehost in "${PW_MAINSITEHOSTS[@]}"; do - MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]" + if [ "${2}" != "maintenance" ] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ]); then + if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ]; then + MSG="Disabling CLS for the Offsite Cluster in vCenter" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 & - echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" - sleep 5s + run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + touch ${PW_TMPFOLDER}/shutdown.cls.offsite else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi - done - echo - [ "${TEST}" != "true" ] && sleep 10s - - elif [ "${ESXIHOST}" == "10.10.2.17" ]; then - echo -e "${idsCL[LightYellow]}Since this is the iSCSI-SYS host server, all iSCSI-SYS VM's are being shutdown ... " - for ehost in "${PW_MAINSITEHOSTS[@]}"; do - MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 & - echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" - sleep 5s - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - done - echo - [ "${TEST}" != "true" ] && sleep 10s - - fi - - MSG="Enabling maintenance mode & starting iSCSI VM migrations" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 & - sleep 25s - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - - MSG="Shutting down remaining VM's" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - if [ "${ESXIHOST}" == "10.10.2.10" ] || [ "${ESXIHOST}" == "10.10.2.17" ]; then - if [ "${ESXIHOST}" == "10.10.2.10" ]; then - run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) - elif [ "${ESXIHOST}" == "10.10.2.17" ]; then - run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) - fi - run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) fi - run=$(${PW_FOLDER}/esxi-scripts/cls-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - - if [ "${2}" != "maintenance" ]; then - MSG="Waiting for VM's to power off or migrate" + if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "MAINSITE" ]; then + MSG="Disabling CLS for the Primary Cluster in vCenter" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + touch ${PW_TMPFOLDER}/shutdown.cls.mainsite + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + fi + MSG="Restarting vCenter CLS Services" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & - echo -e "${idsCL[Green]}Continuing in background\n" + ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 & + echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi + echo + fi - - if [ "${ESXIHOST}" == "10.10.2.10" ]; then - MSG="Waiting for Host ${host} [${ESXIHOST}] to shutdown" + if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ]; then + for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do + if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then + MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then + MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + MSG="Stopping All iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + fi c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" + if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & + fi + sleep 5s + echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + done + echo + fi + + for ESXIHOST in "${ESXI_HOSTS[@]}"; do + MSG="Enabling maintenance mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 & + sleep 10s + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + done + echo + + if [ "${ESXI_HOST}" != "OFFSITE" ] && [ "${ESXI_HOST}" != "ALL" ] && [ "${ESXI_HOST}" != "MAINSITE" ]; then + MSG="Starting iSCSI migrations" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + sleep 20s + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + echo + fi + + if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ]; then + for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do + MSG="Waiting for iSCSI VM's to power down on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1 + elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1 + elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1 + fi + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + done + + MSG="Shutting down iSCSI-Server(s)" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.10 >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.17 >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.10 >/dev/null 2>&1 & + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.17 >/dev/null 2>&1 & + fi + echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + echo + fi + + for ESXIHOST in "${ESXI_HOSTS[@]}"; do + MSG="Shutting down remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & + echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + done + echo + + if [ "${2}" != "maintenance" ]; then + for ESXIHOST in "${ESXI_HOSTS[@]}"; do + MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & + echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + done + echo + + if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + MSG="Waiting for iDS-vMS-iSCSI.scity.us to shutdown" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG//iDS-vMS-iSCSI.scity.us/${idsCL[LightCyan]}iDS-vMS-iSCSI.scity.us}${idsCL[White]}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done + echo -e "${idsCL[Green]}Done${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi MSG="Powering off SAS Enclosure" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2) - sleep 3s - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" + echo -e "${idsCL[Green]}Done${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi - - else - echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be shutdown momentarily ... \n" + fi + [ "${ESXI_HOST}" != "10.10.2.10" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n" + touch ${PW_TMPFOLDER}/shutdown.${ESXI_HOST} else - echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be in maintenance mode momentarily ... \n" + echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be in maintenance mode momentarily ... \n" + touch ${PW_TMPFOLDER}/maintmode.${ESXI_HOST} fi else @@ -2005,23 +1653,23 @@ HOSTMGMT(){ if [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ]; then if [ "${ESXI}" == "maint" ] || [ "${ESXI}" == "maintenance" ] || [ "${ESXI}" == "mm" ]; then echo -e "${idsCL[LightGreen]}PUTTING '${ESXI_HOST}' HOSTS INTO MAINTENANCE MODE${idsCL[Default]}\n" - # SHUTDOWN_SERVERS ${ESXI_HOST,,} maintenance - SHUTDOWN_SERVERS2 ${ESXI_HOST,,} maintenance + # SHUTDOWN_SERVER ${ESXI_HOST,,} maintenance + SHUTDOWN_SERVER ${ESXI_HOST,,} maintenance else echo -e "${idsCL[LightGreen]}SHUTTING DOWN '${ESXI_HOST}' HOSTS${idsCL[Default]}\n" - # SHUTDOWN_SERVERS ${ESXI_HOST,,} - SHUTDOWN_SERVERS2 ${ESXI_HOST,,} + # SHUTDOWN_SERVER ${ESXI_HOST,,} + SHUTDOWN_SERVER ${ESXI_HOST,,} fi else if [ "${ESXI}" == "maint" ] || [ "${ESXI}" == "maintenance" ] || [ "${ESXI}" == "mm" ]; then echo -e "${idsCL[LightGreen]}PUTTING HOST INTO MAINTENANCE MODE${idsCL[Default]}\n" # SHUTDOWN_SERVER ${HOST} maintenance - SHUTDOWN_SERVER2 ${HOST} maintenance + SHUTDOWN_SERVER ${HOST} maintenance else echo -e "${idsCL[LightGreen]}SHUTTING DOWN HOST${idsCL[Default]}\n" # SHUTDOWN_SERVER ${HOST} - SHUTDOWN_SERVER2 ${HOST} + SHUTDOWN_SERVER ${HOST} fi fi @@ -2365,8 +2013,8 @@ fi [ "${2^^}" == "SYS" ] && ISCSISYSVMSHUTDOWN || ISCSIVMSHUTDOWN ;; shutdownhost) SHUTDOWN_SERVER ${2} ${3};; - shutdowntest) SHUTDOWN_SERVER2 ${2} ${3};; - shutdown_servers) SHUTDOWN_SERVERS ${2} ${3};; + shutdowntest) SHUTDOWN_SERVER ${2} ${3};; + shutdown_servers) SHUTDOWN_SERVER ${2} ${3};; hostpower) HOSTPOWER ${2} ${3};; host) HOSTMGMT ${2} ${3} ${4} ${5} ${6} ${7};; diff --git a/shutdown-test.sh b/shutdown-test.sh index f80befc5..63dbef25 100644 --- a/shutdown-test.sh +++ b/shutdown-test.sh @@ -1,4 +1,4 @@ -SHUTDOWN_SERVER2(){ +SHUTDOWN_SERVER(){ if [ "${1}" != "-h" ]; then HOST=${1}; MN=0 if [ "${PW_iDRACHOST_SHORTNAMES[${HOST}]}" != "" ]; then