diff --git a/powerwall.sh b/powerwall.sh index 1c364152..23667ad3 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -56,6 +56,7 @@ CHECKTEMP_SERVICE(){ echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit + echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 3 | read temp_crit_sys temp_diff=$((${last_temp%.*} - ${temp_f%.*})) [ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1)) @@ -68,40 +69,25 @@ CHECKTEMP_SERVICE(){ fi echo "($datetime) WARNING: ${temp_f}°F - (Difference of ${temp_diff}°)" - elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then + elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then echo "($datetime) CRITICAL: ${temp_f}°F - (Difference of ${temp_diff}°)" if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then echo -e "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down servers!!" | mail -s "${1} TEMP CRITICAL" $email_alert echo "alert sent" last_temp=$temp_f - - touch $FOLDER/shutdown - - for ESXIHOST in "${MAINHOSTS[@]}"; do - echo "Shutting Down: ${ESXIHOST}" - /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" - /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh" - /usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" - /usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" - done - for ESXIHOST in "${MAINHOSTS[@]}"; do - echo -en "Waiting for Host (${ESXIHOST}) to shutdown... " - while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do - sleep 1 - done - echo "Ok" - done - - /usr/bin/ssh root@10.10.2.10 "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" - echo -en "Waiting for iSCSI-Server to shutdown... " - while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do - sleep 1 - done - echo "Ok" - echo -e "($datetime) All servers have been shutdown" | mail -s "SERVERS SHUTDOWN" $email_alert + SHUTDOWN_SERVERS main + echo -e "($datetime) Main servers have been shutdown" | mail -s "SERVERS SHUTDOWN" $email_alert fi - + elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then + echo "($datetime) SYSTEM CRITICAL: ${temp_f}°F - (Difference of ${temp_diff}°)" + if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then + echo -e "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down system servers!!" | mail -s "${1} TEMP CRITICAL" $email_alert + echo "alert sent" + last_temp=$temp_f + SHUTDOWN_SERVERS sys + echo -e "($datetime) System servers have been shutdown\nAll servers are now offline" | mail -s "SERVERS SHUTDOWN" $email_alert + fi else if [ $last_temp -gt 1 ]; then echo -e "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp}°F" | mail -s "${1} BACK TO NORMAL" $email_alert @@ -154,6 +140,33 @@ UPDATE(){ fi } +SHUTDOWN_SERVERS(){ + touch $FOLDER/shutdown + + HOSTS=${1}HOSTS[@] + for ESXIHOST in "${!HOSTS}"; do + echo "Shutting Down: ${ESXIHOST}" + /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" + /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh" + /usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" + /usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" + done + for ESXIHOST in "${MAINHOSTS[@]}"; do + echo -en "Waiting for Host (${ESXIHOST}) to shutdown... " + while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do + sleep 1 + done + echo "Ok" + done + + /usr/bin/ssh root@10.10.2.10 "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" + echo -en "Waiting for iSCSI-Server to shutdown... " + while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do + sleep 1 + done + echo "Ok" +} +