This commit is contained in:
2023-08-12 10:22:00 -05:00
parent a4ab994219
commit 98a0613cd0
2 changed files with 138 additions and 102 deletions

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
VERS='2.161-08122023'
VERS='2.162-08122023'
noheader=' update service dailytemp '

View File

@@ -59,8 +59,11 @@ CHECKTEMPSENSOR(){
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
elif [ "${SENSOR_TYPE[${1}]}" == "remotesystem" ]; then
temp_c=$(ssh root@${REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
if [ "$(CHECK_HOST ${REMOTE_SENSORS[${1}]})" != "false" ]; then
temp_c=$(ssh root@${REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
fi
fi
@@ -117,112 +120,145 @@ CHECKTEMP_SERVICE(){
errtime=$(expr `date +%s` - $(stat -c %Y ${logtemp}))
[ $errtime -ge 3600 ] && relog=1 || relog=0
if [ "${SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
elif [ "${SENSOR_TYPE[${SENSOR}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
fi
datetime=`date +'%Y-%m-%d %H:%M:%S'`
if [ "$(CHECK_HOST ${REMOTE_SENSORS[${1}]})" != "false" ]; then
if [ -f ${TMPFOLDER}/${SENSOR}.down ]; then
SENDNOTICE "${SENSOR} Online" "${SENSOR} is back online"
echo "($datetime) - ${SENSOR} - ${SENSOR} is back onlione" >> ${logtemp}
rm -f ${TMPFOLDER}/${SENSOR}.down
fi
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
if [ "${SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
elif [ "${SENSOR_TYPE[${SENSOR}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
elif [ "${SENSOR_TYPE[${SENSOR}]}" == "remotesystem" ]; then
temp_c=$(ssh root@${REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
fi
datetime=`date +'%Y-%m-%d %H:%M:%S'`
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
if [ -f ${TMPFOLDER}/${SENSOR}-error.reading ]; then
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
echo "($datetime) - ${SENSOR} - ${SENSOR} is reporting data again" >> ${logtemp}
rm -f ${TMPFOLDER}/${SENSOR}-error.reading
fi
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
if [ "${last_temp[${SENSOR}]}" != "" ]; then
temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
else
last_temp[${SENSOR}]=0
temp_diff=0
fi
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
touch ${TMPFOLDER}/temp.warn
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
last_temp[${SENSOR}]=$temp_f
echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
touch ${TMPFOLDER}/temp.crit
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
touch ${TMPFOLDER}/temp.critsys
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down system servers!!" 1
echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
else
[ ! -f ${TMPFOLDER}/temp.* ] && [ $(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/.${SENSOR}.sentdaily)) -gt 600 ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ] && touch ${TMPFOLDER}/.${SENSOR}.sentdaily && SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)" &
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${logfile}
[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
NORMAL TEMP: ${temp_f}'F"
echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${logfile}
[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
elif [ $relog -eq 1 ]; then
echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp}
fi
rm -f ${TMPFOLDER}/temp.*
last_temp[${SENSOR}]=1
fi
[ -f ${TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
if [ "$(compgen -G "${TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then
wait=60
else
wait=300
fi
if [ ${lastinsert} -ge ${wait} ]; then
if [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
if [ "${last_temp[${SENSOR}]}" != "" ]; then
temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
last_temp[${SENSOR}]=0
temp_diff=0
fi
${mysql_conn} -e "${QRY}"
touch ${TMPFOLDER}/${SENSOR}.insert
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
touch ${TMPFOLDER}/temp.warn
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
last_temp[${SENSOR}]=$temp_f
echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
touch ${TMPFOLDER}/temp.crit
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
touch ${TMPFOLDER}/temp.critsys
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down system servers!!" 1
echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
else
[ ! -f ${TMPFOLDER}/temp.* ] && [ $(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/.${SENSOR}.sentdaily)) -gt 600 ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ] && touch ${TMPFOLDER}/.${SENSOR}.sentdaily && SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)" &
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${logfile}
[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
NORMAL TEMP: ${temp_f}'F"
echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${logfile}
[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
elif [ $relog -eq 1 ]; then
echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp}
fi
rm -f ${TMPFOLDER}/temp.*
last_temp[${SENSOR}]=1
fi
[ -f ${TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
if [ "$(compgen -G "${TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then
wait=60
else
wait=300
fi
if [ ${lastinsert} -ge ${wait} ]; then
if [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
fi
${mysql_conn} -e "${QRY}"
touch ${TMPFOLDER}/${SENSOR}.insert
fi
else
if [ ! -f ${TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading sensor data" 1
echo "($datetime) - ${SENSOR} - ERROR reading sensor data" >> ${logtemp}
touch ${TMPFOLDER}/${SENSOR}-error.reading
fi
fi
else
SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1
echo "($datetime) - alert sent" >> ${logtemp}
if [ ! -f ${TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
SENDNOTICE "${SENSOR} ERROR" "${SENSOR} is down" 1
echo "($datetime) - ${SENSOR} - ${SENSOR} is down" >> ${logtemp}
touch ${TMPFOLDER}/${SENSOR}.down
fi
fi
done