From 552bd90054efc1ef6deed1977af37d727819448a Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Mon, 11 Dec 2023 15:34:54 -0600 Subject: [PATCH] update --- offsite-power-check.sh | 3 + powerwall.sh | 204 ++++++++++++++++++++--------------------- 2 files changed, 102 insertions(+), 105 deletions(-) diff --git a/offsite-power-check.sh b/offsite-power-check.sh index 595ed675..f2d20187 100755 --- a/offsite-power-check.sh +++ b/offsite-power-check.sh @@ -97,6 +97,9 @@ $(date)" for host in ${PW_OFFSITEHOSTS_IDRAC}; do if [ "${TEST}" == "" ] || [ "${TEST}" == "full" ]; then ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on & + run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) else echo "TESTMODE: Powering up '${host}'" fi diff --git a/powerwall.sh b/powerwall.sh index e4dbdc68..6c5422be 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -240,22 +240,6 @@ CHECKTEMP_SERVICE(){ [ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off t=1 - # # iDS-vMS-Offsite Fan Issue Workaround - # if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then - # # vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq) - # # FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'` - # vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1) - # if [ "${vHOSTFAN}" != "" ]; then - # FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') - # if [ ${FanSpeed} -gt 92 ]; then - # SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Workaround commands sent to set the Fans at 30%25" 1 - # ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1 - # ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1 - # FanSpeed=30 - # fi - # fi - # fi - else SENSORS_CHECK=(${SENSORa}) fi @@ -327,15 +311,10 @@ CHECKTEMP_SERVICE(){ echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys fi - - if [ "${last_temp[${SENSOR}]}" != "" ]; then - temp_diff=`echo "scale=2; ${last_temp[${SENSOR}]}-${temp_f}" | bc` - [ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc` - - else - last_temp[${SENSOR}]=0 - temp_diff=0 - fi + + [ ! "${last_temp[${SENSOR}]}" ] && last_temp[${SENSOR}]=${temp_f} + temp_diff=`echo "scale=2; ${last_temp[${SENSOR}]}-${temp_f}" | bc` + [ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc` if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then touch ${PW_TMPFOLDER}/temp.warn @@ -360,27 +339,30 @@ CHECKTEMP_SERVICE(){ echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} last_temp[${SENSOR}]=${temp_f} - if [[ "${PW_OFFSITEHOSTS}" != *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then + if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f touch ${PW_TMPFOLDER}/temp.crit ]; then + touch ${PW_TMPFOLDER}/temp.critsys SHUTDOWN_MAIN ${SENSOR} & echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp} + + # elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then + # SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1 + # SHUTDOWN_SERVER ${SENSORa} & + # touch ${PW_TMPFOLDER}/power.${SENSORa}.off - elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then - SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1 - SHUTDOWN_SERVER ${SENSORa} & - touch ${PW_TMPFOLDER}/power.${SENSORa}.off - - fi + fi fi elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then - touch ${PW_TMPFOLDER}/temp.critsys if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} last_temp[${SENSOR}]=${temp_f} - SHUTDOWN_SYS ${SENSOR} TEMP & - SHUTDOWN_CRIT ${SENSOR} & - echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp} + if [ ! -f touch ${PW_TMPFOLDER}/temp.critsys ]; then + touch ${PW_TMPFOLDER}/temp.critsys + [ "${SENSOR}" == "ServerRoomTH" ] && SHUTDOWN_SYS ${SENSOR} TEMP & + SHUTDOWN_CRIT ${SENSOR} & + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp} + fi fi else if [ $(bc -l <<< "${last_temp[${SENSOR}]} > 1") -eq 1 ]; then @@ -388,14 +370,14 @@ CHECKTEMP_SERVICE(){ Previous Temp: ${last_temp[${SENSOR}]}${GAUGESH}" echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR}]}${GAUGESH}" >> ${logtemp} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE} - [ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.* + # [ "$(compgen -G "${PW_TMPFOLDER}/shutdown.*")" != "" ] && rm -f ${PW_TMPFOLDER}/shutdown.* elif [ "${last_temp[${SENSOR}]}" == "1" ]; then # SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup # NORMAL TEMP: ${temp_f}${GAUGESH}" echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE} - [ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.* + # [ "$(compgen -G "${PW_TMPFOLDER}/shutdown.*")" != "" ] && rm -f ${PW_TMPFOLDER}/shutdown.* elif [ "${relog}" == "1" ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp} @@ -844,8 +826,8 @@ CHECKPOWER(){ } CHECKPOWER_SERVICE(){ - voltstatus=0 - battstatus=0 + touch ${PW_TMPFOLDER}/power.ac.start + touch ${PW_TMPFOLDER}/power.dc.start while true; do INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json") SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //') @@ -857,6 +839,10 @@ CHECKPOWER_SERVICE(){ elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1 + + ######################### FINISH + + fi elif [ -f ${PW_TMPFOLDER}/power.ups.low ]; then @@ -866,16 +852,17 @@ CHECKPOWER_SERVICE(){ if [ "${INVSTATSINFO}" != "" ]; then INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV') BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV') + [ ! "${last_battvolt}" ] && last_battvolt=${BATTVOLT} if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then - if [ ${voltstatus} -eq 3 ]; then + [ "${last_battvolt}" != "${BATTVOLT}" ] && last_battvolt=${BATTVOLT} + if [ -f ${PW_TMPFOLDER}/power.ac.low ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Back to Normal Voltage" >> ${PW_LOGPOWER} SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected VOLTAGE: ${INPUTACV}'volts" - battstatus=0 else [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts - Normal Voltage" >> ${PW_LOGPOWER} - if [ ${voltstatus} -eq 0 ]; then + if [ -f ${PW_TMPFOLDER}/power.ac.start ]; then SENDNOTICE "Power Nominal" "Service Startup Normal voltage detected VOLTAGE: ${INPUTACV}'volts @@ -884,27 +871,27 @@ BATT VOLTAGE: ${BATTVOLT}'volts" echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - 'Check Power' Service Startup - Normal voltage detected" >> ${PW_LOGPOWER} fi fi - voltstatus=1 - rm -f ${PW_TMPFOLDER}/power.ac.* + if [ ! -f ${PW_TMPFOLDER}/power.ac.good ]; then + rm -f ${PW_TMPFOLDER}/power.ac.* + touch ${PW_TMPFOLDER}/power.ac.good + fi # UPDATE APC-PDU LINE TO LINE VOLTAGE CURRENT_VOLTAGE=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 | sed 's/.*: //') SET_VOLTAGE=`echo "scale=2; $(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')/10" | bc | awk '{print int($1+0.5)}'` - [ ${CURRENT_VOLTAGE} -ne ${SET_VOLTAGE} ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 i ${SET_VOLTAGE} >/dev/null 2>&1) && echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${CURRENT_VOLTAGE}'volts --> ${SET_VOLTAGE}'volts - LINE-TO-LINE VOLTAGE UPDATED ON APC-PDU" >> ${PW_LOGPOWER} + [ "${SET_VOLTAGE}" ] && [ ${CURRENT_VOLTAGE} -ne ${SET_VOLTAGE} ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 i ${SET_VOLTAGE} >/dev/null 2>&1) && echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${CURRENT_VOLTAGE}'volts --> ${SET_VOLTAGE}'volts - LINE-TO-LINE VOLTAGE UPDATED ON APC-PDU" >> ${PW_LOGPOWER} else - if [ ${voltstatus} -lt 3 ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW voltage detected" >> ${PW_LOGFILE} + [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW" >> ${PW_LOGPOWER} + if [ -f ${PW_TMPFOLDER}/power.ac.good ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW voltage detected" >> ${PW_LOGPOWER} SENDNOTICE "POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected VOLTAGE: ${INPUTACV}'volts BATT VOLTAGE: ${BATTVOLT}'volts" 1 - last_battvolt=${BATTVOLT} - voltstatus=3 +rm -f ${PW_TMPFOLDER}/power.ac.good touch ${PW_TMPFOLDER}/power.ac.low fi - if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge 180 ]; then - echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGFILE} + if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge 120 ]; then echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGPOWER} SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1 SHUTDOWN_SERVER ${PW_HOST_IDRACS['iDS-vMS-Host0']} & @@ -912,50 +899,56 @@ BATT VOLTAGE: ${BATTVOLT}'volts" 1 fi fi - if (( $(bc <<<"${BATTVOLT} < ${min_battvolt} && ${BATTVOLT} > ${min_battvolt_sys}") )); then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER} - echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE} - touch ${PW_TMPFOLDER}/power.dc.low - if (( $(bc <<<"${battstatus} < 2 && ${INPUTACV} < ${min_acvolt}") )); then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER} - SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts -Shutting down main servers" 1 - SHUTDOWN_MAIN POWER & - battstatus=2 - fi - - elif (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE} - touch ${PW_TMPFOLDER}/power.dc.low - if (( $(bc <<<"${battstatus} < 3 && ${INPUTACV} < ${min_acvolt}") )); then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGFILE} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER} - SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts -Shutting down all servers" 1 - SHUTDOWN_SYS POWER & - - battstatus=3 - - fi - - else + if (( $(bc <<<"${BATTVOLT} >= ${min_battvolt}") )); then [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage" >> ${PW_LOGPOWER} - if [ ${battstatus} -eq 0 ]; then + if [ -f ${PW_TMPFOLDER}/power.dc.start ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER} + rm -f ${PW_TMPFOLDER}/power.dc.start fi - battstatus=1 - + if [ ! -f ${PW_TMPFOLDER}/power.dc.good ]; then + rm -f ${PW_TMPFOLDER}/power.dc.* + touch ${PW_TMPFOLDER}/power.dc.good + fi + + else + [ -f ${PW_TMPFOLDER}/power.dc.good ] && rm -f ${PW_TMPFOLDER}/power.dc.good + [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER} + + if (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then + [ ! -f ${PW_TMPFOLDER}/power.dc.critlow ] && touch ${PW_TMPFOLDER}/power.dc.critlow + + if [ ! -f ${PW_TMPFOLDER}/power.sys.off ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )); then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER} + SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts +Shutting down all remaining servers" 1 + SHUTDOWN_SYS POWER & + + fi + + elif (( $(bc <<<"${BATTVOLT} < ${min_battvolt}") )); then + [ ! -f ${PW_TMPFOLDER}/power.dc.low ] && touch ${PW_TMPFOLDER}/power.dc.low + + if [ ! -f ${PW_TMPFOLDER}/power.main.off ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )); then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER} + SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts +Shutting down main servers" 1 + SHUTDOWN_MAIN POWER & + + fi + fi + + volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc` + (( $(bc <<<"${volt_diff} < 0") )) && volt_diff=`echo "scale=1; ${volt_diff} * -1" | bc` + if (( $(bc <<<"${volt_diff} > 0.5") )); then + SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts" + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER} + last_battvolt=${BATTVOLT} + fi + fi - volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc` - (( $(bc <<<"${volt_diff} < 0") )) && temp_diff=`echo "scale=1; ${volt_diff} * -1" | bc` - if (( $(bc <<<"${volt_diff} > 0.5") )); then - SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts" - echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER} - last_battvolt=${BATTVOLT} - fi else sleep 20s fi @@ -1212,7 +1205,7 @@ SHUTDOWN_SYS(){ if [ "${sendnotice}" == "true"]; then SENDNOTICE "SYS SERVERS SHUTDOWN" "SYS servers have been shutdown - ${reason}" 1 +${reason}" 1 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp} fi @@ -1243,12 +1236,12 @@ SHUTDOWN_CRIT(){ fi } SHUTDOWN_OFFSITE(){ - if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then + if [ ! -f ${PW_TMPFOLDER}/shutdown.offsite ]; then SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down offsite host(s)" 1 for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do SHUTDOWN_SERVER ${offsite_host} & done - touch ${PW_FOLDER}/shutdown.offsite + touch ${PW_TMPFOLDER}/shutdown.offsite fi } @@ -1257,8 +1250,8 @@ SHUTDOWN_SERVERS(){ [ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE} if [ "${1^^}" == "MAIN" ]; then - if [ ! -f ${PW_FOLDER}/shutdown.main ]; then - touch ${PW_FOLDER}/shutdown.main + if [ ! -f ${PW_TMPFOLDER}/power.main.off ]; then + touch ${PW_TMPFOLDER}/power.main.off if [ "${2}" == "TEMP" ] && [ "${2}" != "maintenance" ]; then MSG="Disabling CLS in vCenter" @@ -1365,9 +1358,9 @@ SHUTDOWN_SERVERS(){ fi elif [ "${1^^}" == "SYS" ]; then - [ ! -f ${PW_FOLDER}/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN TEMP - if [ ! -f ${PW_FOLDER}/shutdown.sys ]; then - touch ${PW_FOLDER}/shutdown.sys + [ ! -f ${PW_TMPFOLDER}/power.main.off ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN TEMP + if [ ! -f ${PW_TMPFOLDER}/power.sys.off ]; then + touch ${PW_TMPFOLDER}/power.sys.off if [ "${2}" == "TEMP" ] && [ "${2}" != "maintenance" ]; then MSG="Disabling CLS in vCenter" @@ -1450,10 +1443,11 @@ SHUTDOWN_SERVERS(){ fi elif [ "${1^^}" == "ALL" ]; then - if [ ! -f ${PW_FOLDER}/shutdown.all ]; then - touch ${PW_FOLDER}/shutdown.all + if [ ! -f ${PW_TMPFOLDER}/power.all.off ]; then + touch ${PW_TMPFOLDER}/power.all.off if [ "${2}" != "maintenance" ]; then + MSG="Disabling CLS in vCenter" c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " @@ -1559,8 +1553,8 @@ SHUTDOWN_SERVERS(){ fi elif [ "${1^^}" == "MAINSITE" ]; then - if [ ! -f ${PW_FOLDER}/shutdown.mainsite ]; then - touch ${PW_FOLDER}/shutdown.mainsite + if [ ! -f ${PW_TMPFOLDER}/power.main.offsite ]; then + touch ${PW_TMPFOLDER}/power.main.offsite if [ "${2}" == "TEMP" ] && [ "${2}" != "maintenance" ]; then MSG="Disabling CLS in vCenter" @@ -1667,8 +1661,8 @@ SHUTDOWN_SERVERS(){ fi elif [ "${1^^}" == "OFFSITE" ]; then - if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then - touch ${PW_FOLDER}/shutdown.offsite + if [ ! -f ${PW_TMPFOLDER}/shutdown.offsite ]; then + touch ${PW_TMPFOLDER}/shutdown.offsite if [ "${2}" != "maintenance" ]; then MSG="Disabling CLS in vCenter"