This commit is contained in:
2023-05-13 09:46:55 -05:00
parent e093c50357
commit f9f86debf1
2 changed files with 24 additions and 16 deletions

View File

@@ -1,5 +1,5 @@
VERS='1.108-05122023'
VERS='1.109-05122023'
noupdate=' stop start _service '
@@ -41,3 +41,11 @@ MAINHOSTS=(${MAINHOSTS})
SYSHOSTS=(${SYSHOSTS})
unset IFS
SENDNOTICE(){
[ "${PUSHOVER_APP_TOKEN}" != "" ] && PUSH_TO_MOBILE "${2}
$(date)" "${1}" ${3} &
[ "${EMAIL_NOTICE}" != "" ] && echo -e "${2}\n\n$(date)" | mail -s "${1}" ${EMAIL_NOTICE}
}

View File

@@ -67,7 +67,7 @@ CHECKTEMP_SERVICE(){
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
echo "($datetime) WARNING TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)" | mail -s "${1} TEMP WARNING" $email_alert
SENDNOTICE "${1} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)" 1
echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc}
last_temp=$temp_f
fi
@@ -76,31 +76,31 @@ CHECKTEMP_SERVICE(){
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
echo "($datetime) - ${temp_f} F - CRITICAL - (Difference of ${temp_diff} F)" >> ${logsrvc}
if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
echo -e "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down servers!!" | mail -s "${1} TEMP CRITICAL" $email_alert
SENDNOTICE "${1} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down servers!!" 1
echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc}
last_temp=$temp_f
# SHUTDOWN_SERVERS MAIN
echo -e "($datetime) Main servers have been shutdown" | mail -s "SERVERS SHUTDOWN" $email_alert
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1
echo -e "($datetime) - ${temp_f}F - Main servers have been shutdown" >> ${logsrvc}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
echo "($datetime) - ${temp_f}F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logsrvc}
if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
echo -e "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down system servers!!" | mail -s "${1} TEMP CRITICAL" $email_alert
SENDNOTICE "${1} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down system servers!!" 1
echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc}
last_temp=$temp_f
# SHUTDOWN_SERVERS SYS
echo -e "($datetime) System servers have been shutdown" | mail -s "SERVERS SHUTDOWN" $email_alert
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1
echo -e "($datetime) - ${temp_f}F - System servers have been shutdown" >> ${logsrvc}
fi
else
if [ $last_temp -gt 1 ]; then
echo -e "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp}°F" | mail -s "${1} BACK TO NORMAL" $email_alert
SENDNOTICE "${1} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp}°F"
echo -e "($datetime) - ${temp_f}F - Back to NORMAL TEMP - Previous Temp: ${last_temp}F" >> ${logsrvc}
echo "($datetime) alert sent" >> ${logsrvc}
elif [ $last_temp -eq 0 ]; then
echo -e "($datetime) Service Startup\n NORMAL TEMP: ${temp_f}°F" | mail -s "${1} TEMP NORMAL" $email_alert;
SENDNOTICE "${1} TEMP NORMAL" "($datetime) Service Startup\n NORMAL TEMP: ${temp_f}°F"
echo -e "($datetime) - ${temp_f}F - Service Startup - NORMAL TEMP" >> ${logsrvc}
fi
last_temp=1
@@ -111,8 +111,8 @@ CHECKTEMP_SERVICE(){
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[$1]}','${datetime}','${temp_f}','${temp_h}')"
${mysql_conn} -e "${QRY}"
else
echo "($datetime) ERROR reading temperature" | mail -s "${1} ERROR" $email_alert
[ $VERBOSE = true ] && echo "($datetime) alert sent"
SENDNOTICE "${1} ERROR" "($datetime) ERROR reading temperature" 1
echo "($datetime) alert sent" >> ${logsrvc}
fi
done
sleep 10
@@ -146,14 +146,14 @@ CHECKACV_SERVICE(){
if [ ${INPUTACV} -ge ${min_acvolt} ]; then
if [ $voltstatus -eq 3 ]; then
echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${logacv}
echo -e "($datetime) Normal voltage detected\nVOLTAGE: ${INPUTACV}V" | mail -s "POWER NOMINAL AGAIN" $email_alert
SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected\nVOLTAGE: ${INPUTACV}V"
echo "($datetime) alert sent" >> ${logacv}
battstatus=0
else
errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
[ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${logacv}
if [ $voltstatus -eq 0 ]; then
echo -e "($datetime) Service Startup\nNormal voltage detected\nVOLTAGE: ${INPUTACV}V" | mail -s "Power Nominal" $email_alert
SENDNOTICE "Power Nominal" "($datetime) Service Startup\nNormal voltage detected\nVOLTAGE: ${INPUTACV}V"
echo "($datetime) - ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${logacv}
fi
fi
@@ -161,7 +161,7 @@ CHECKACV_SERVICE(){
else
echo "($datetime) - ${INPUTACV}V - LOW Voltage" >> ${logacv}
if [ $voltstatus -lt 3 ]; then
echo -e "($datetime) Power off or low voltage detected\nVOLTAGE: ${INPUTACV}V\nBATT VOLTAGE: ${BATTVOLT}V" | mail -s "POWER ALERT - LOW POWER!!" $email_alert
SENDNOTICE "POWER ALERT - LOW POWER!!" "($datetime) Power off or low voltage detected\nVOLTAGE: ${INPUTACV}V\nBATT VOLTAGE: ${BATTVOLT}V" 1
echo "($datetime) alert sent" >> ${logacv}
last_battvolt=$BATTVOLT
fi
@@ -170,7 +170,7 @@ CHECKACV_SERVICE(){
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
if [ $battstatus -lt 2 ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logacv}
echo -e "($datetime) Battery Voltage LOW: ${BATTVOLT}V\nShutting down main servers" | mail -s "BATT-VOLT LOW: SHUTDOWN SERVERS" $email_alert
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V\nShutting down main servers" 1
# SHUTDOWN_SERVERS MAIN
battstatus=2
fi
@@ -179,7 +179,7 @@ CHECKACV_SERVICE(){
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
if [ $battstatus -lt 3 ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logacv}
echo -e "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V\nShutting down all servers" | mail -s "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" $email_alert
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V\nShutting down all servers" 1
# SHUTDOWN_SERVERS SYS
curl --data "cmd=PWRoff" http://10.10.0.61/cmd
@@ -196,7 +196,7 @@ CHECKACV_SERVICE(){
volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*}))
[ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1))
if [ $volt_diff -gt 0 ]; then
echo -e "($datetime) Battery Voltage: ${BATTVOLT}V" | mail -s "BATTERY VOLTAGE CHANGE" $email_alert
SENDNOTICE "BATTERY VOLTAGE CHANGE" "($datetime) Battery Voltage: ${BATTVOLT}V"
echo -e "($datetime) - ${BATTVOLT}V - Battery Voltage Change" >> ${logacv}
last_battvolt=$BATTVOLT
fi