From 62b4d3976d34911dfd9ac11d08c056e3b654ae95 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Wed, 20 Dec 2023 08:44:43 -0600 Subject: [PATCH] update --- defaults.inc | 2 +- inc/power.inc.sh | 10 ++--- powerwall.sh | 113 ++++++++++++++++++++++++++++------------------- 3 files changed, 74 insertions(+), 51 deletions(-) diff --git a/defaults.inc b/defaults.inc index efa6fa6b..62fb23d0 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ -VERS='2.5.48-12192023' +VERS='2.5.49-12202023' noheader=' update service dailytemp confsync ' diff --git a/inc/power.inc.sh b/inc/power.inc.sh index 5ea9c960..84fed050 100644 --- a/inc/power.inc.sh +++ b/inc/power.inc.sh @@ -392,7 +392,7 @@ BATT VOLTAGE: ${BATTVOLT}'volts" touch ${PW_TMPFOLDER}/power.ac.good elif [ -f ${PW_TMPFOLDER}/.power.restored ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.power.restored)) -ge $(echo "scale=0; ${PW_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then - HOSTMGMT mainsite -p on ###>/dev/null 2>&1 & + HOSTMGMT mainsite -p on >/dev/null 2>&1 & rm -f ${PW_TMPFOLDER}/.power.restored fi @@ -415,7 +415,7 @@ BATT VOLTAGE: ${BATTVOLT}'volts" 1 if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_CONSERVE_AFTER}*60/1" | bc) ]; then echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGPOWER} SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1 - SHUTDOWN_SERVER ${PW_MAIN_ISCSI} -esxi off ###& + SHUTDOWN_SERVER ${PW_MAIN_ISCSI} -esxi off & touch ${PW_TMPFOLDER}/power.ac.conserve fi fi @@ -443,7 +443,7 @@ BATT VOLTAGE: ${BATTVOLT}'volts" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER} SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts Shutting down all remaining servers" 1 - SHUTDOWN_SYS POWER ###& + SHUTDOWN_SYS POWER & fi @@ -456,7 +456,7 @@ Shutting down all remaining servers" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER} SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts Shutting down main servers" 1 - SHUTDOWN_MAIN POWER ###& + SHUTDOWN_MAIN POWER & fi @@ -465,7 +465,7 @@ Shutting down main servers" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER} SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts Shutting down main servers" 1 - SHUTDOWN_MAIN POWER ###& + SHUTDOWN_MAIN POWER & fi fi diff --git a/powerwall.sh b/powerwall.sh index 8ba8f066..4f6f44b1 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -245,15 +245,24 @@ SHUTDOWN_SERVER(){ c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & - elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & - elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & + if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" == "" ]; then + hostonline=0 + else + hostonline=1 + fi + if [ ${hostonline} -eq 1 ]; then + if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & + elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & + fi + sleep 5s + echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" fi - sleep 5s - echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi @@ -286,7 +295,6 @@ SHUTDOWN_SERVER(){ echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi fi - MSG="Restarting vCenter CLS Services" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " @@ -304,26 +312,33 @@ SHUTDOWN_SERVER(){ c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 & - sleep 10s - echo -e "${idsCL[Green]}Done${idsCL[Default]}" + if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then + ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 & + sleep 10s + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" + fi else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi done echo - - if [ "${ESXI_HOST}" != "OFFSITE" ] && [ "${ESXI_HOST}" != "ALL" ] && [ "${ESXI_HOST}" != "MAINSITE" ]; then - MSG="Starting iSCSI migrations" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - sleep 30s - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + if [ ${MM} -eq 0 ] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then + tmp=tmp + else + if [ "${ESXI_HOST}" != "OFFSITE" ] && [ "${ESXI_HOST}" != "ALL" ] && [ "${ESXI_HOST}" != "MAINSITE" ]; then + MSG="Starting iSCSI migrations" + c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done + echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " + if [ "${TEST}" != "true" ]; then + sleep 35s + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" + fi + echo fi - echo fi if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ]; then @@ -332,10 +347,10 @@ SHUTDOWN_SERVER(){ echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do if [ "${TEST}" != "true" ]; then - if [ "$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then + if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then VERIFY_ISCSI_VM_DOWN "${ESXI_HOST}" "${ESXIHOST}" & else - touch ${PW_TMPFOLDER}/verify.iscsi.${ESXI_HOST}.done + touch ${PW_TMPFOLDER}/verify.iscsi.${ESXIHOST}.done fi fi done @@ -344,9 +359,9 @@ SHUTDOWN_SERVER(){ nc_count=0 until [ ${nc_count} -eq ${#PW_MAINSITEHOSTS[@]} ]; do for iscsistatus in ${PW_TMPFOLDER}/verify.iscsi.*.done; do - NTS=$(grep -oP '(?<=iscsi.).*?(?=.done)' <<< "${iscsistatus}") - if [ "${NTS}" != "*" ]; then - rm -f ${STATUSRUN_TMPFOLDER}/status-check.${NTS}.done + NICC=$(grep -oP '(?<=iscsi.).*?(?=.done)' <<< "${iscsistatus}") + if [ "${NICC}" != "*" ]; then + rm -f ${STATUSRUN_TMPFOLDER}/status-check.${NICC}.done ((nc_count++)) fi done @@ -359,7 +374,7 @@ SHUTDOWN_SERVER(){ MSG="Shutting down iSCSI-Server(s)" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then + if [ "${TEST}" != "true" ]; then if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.10 >/dev/null 2>&1 & elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then @@ -379,9 +394,13 @@ SHUTDOWN_SERVER(){ MSG="Shutting down remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & - echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" + if [ "${TEST}" != "true" ]; then + if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then + ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & + echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" + fi else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi @@ -393,21 +412,25 @@ SHUTDOWN_SERVER(){ MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & - echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" + if [ "${TEST}" != "true" ]; then + if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then + MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & + echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" + fi else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi done echo - if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then - MSG="Waiting for iDS-vMS-iSCSI.scity.us to shutdown" + if [ "${ESXI_HOST}" == "${PW_MAIN_ISCSI}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then + MSG="Waiting for ${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]} to shutdown" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[White]}${MSG//iDS-vMS-iSCSI.scity.us/${idsCL[LightCyan]}iDS-vMS-iSCSI.scity.us}${idsCL[White]}${MSG_SPC}: " + echo -en "${idsCL[White]}${MSG//${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}/${idsCL[LightCyan]}${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}}${idsCL[White]}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done + while ping -qw 10 -c3 "${PW_MAIN_ISCSI}">/dev/null 2>&1; do sleep 1; done echo -e "${idsCL[Green]}Done${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" @@ -416,7 +439,7 @@ SHUTDOWN_SERVER(){ MSG="Powering off SAS Enclosure" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then + if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST ${PW_MAIN_ISCSI})" == "false" ]; then run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2) echo -e "${idsCL[Green]}Done${idsCL[Default]}" else @@ -424,7 +447,7 @@ SHUTDOWN_SERVER(){ fi fi - [ "${ESXI_HOST}" != "10.10.2.10" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n" + [ "${ESXI_HOST}" != "${PW_MAIN_ISCSI}" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n" touch ${PW_TMPFOLDER}/shutdown.${ESXI_HOST} else echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be in maintenance mode momentarily ... \n" @@ -439,13 +462,13 @@ SHUTDOWN_SERVER(){ VERIFY_ISCSI_VM_DOWN(){ if [ "${1}" == "10.10.2.10" ] || [ "${1}" == "MAIN" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${2} ###>/dev/null 2>&1 + ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${2} >/dev/null 2>&1 elif [ "${1}" == "10.10.2.17" ] || [ "${1}" == "SYS" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${2} ###>/dev/null 2>&1 + ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${2} >/dev/null 2>&1 elif [ "${1}" == "MAINSITE" ] || [ "${1}" == "ALL" ]; then - ${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 ${2} ###>/dev/null 2>&1 + ${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 ${2} >/dev/null 2>&1 fi - touch ${PW_TMPFOLDER}/verify.iscsi.${1}.done + touch ${PW_TMPFOLDER}/verify.iscsi.${2}.done } HOSTMGMT(){