From 98a0613cd04a2d3a49f70730a5b1f44764951bce Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Sat, 12 Aug 2023 10:22:00 -0500
Subject: [PATCH] update

---
 defaults.inc |   2 +-
 powerwall.sh | 238 +++++++++++++++++++++++++++++----------------------
 2 files changed, 138 insertions(+), 102 deletions(-)

diff --git a/defaults.inc b/defaults.inc
index 48c363d7..680640a9 100755
--- a/defaults.inc
+++ b/defaults.inc
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-VERS='2.161-08122023'
+VERS='2.162-08122023'
 
 noheader=' update service dailytemp '
 
diff --git a/powerwall.sh b/powerwall.sh
index 1a4b01df..79b6596a 100755
--- a/powerwall.sh
+++ b/powerwall.sh
@@ -59,8 +59,11 @@ CHECKTEMPSENSOR(){
 		temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
 		
 	elif [ "${SENSOR_TYPE[${1}]}" == "remotesystem" ]; then
-		temp_c=$(ssh root@${REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
-		temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
+		if [ "$(CHECK_HOST ${REMOTE_SENSORS[${1}]})" != "false" ]; then
+			temp_c=$(ssh root@${REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
+			temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
+			
+		fi
 		
 	fi
 	
@@ -117,112 +120,145 @@ CHECKTEMP_SERVICE(){
 			errtime=$(expr `date +%s` - $(stat -c %Y ${logtemp}))
 			[ $errtime -ge 3600 ] && relog=1 || relog=0
 			
-			if [ "${SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then
-				mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
-				echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
-				echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
-				
-			elif [ "${SENSOR_TYPE[${SENSOR}]}" == "system" ]; then
-				temp_c=$(vcgencmd measure_temp)
-				temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
-				temp_h=''
-			fi
-			datetime=`date +'%Y-%m-%d %H:%M:%S'`
+			if [ "$(CHECK_HOST ${REMOTE_SENSORS[${1}]})" != "false" ]; then
+				if [ -f ${TMPFOLDER}/${SENSOR}.down ]; then
+					SENDNOTICE "${SENSOR} Online" "${SENSOR} is back online"
+					echo "($datetime) - ${SENSOR} - ${SENSOR} is back onlione" >> ${logtemp}
+					rm -f ${TMPFOLDER}/${SENSOR}.down
+					
+				fi
 			
-			if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
-				temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
+				if [ "${SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then
+					mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
+					echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
+					echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
+				
+				elif [ "${SENSOR_TYPE[${SENSOR}]}" == "system" ]; then
+					temp_c=$(vcgencmd measure_temp)
+					temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
+					temp_h=''
+				
+				elif [ "${SENSOR_TYPE[${SENSOR}]}" == "remotesystem" ]; then
+					temp_c=$(ssh root@${REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
+					temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
+					temp_h=''
+				
+				fi
+				datetime=`date +'%Y-%m-%d %H:%M:%S'`
+			
+				if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
+					if [ -f ${TMPFOLDER}/${SENSOR}-error.reading ]; then
+						SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
+						echo "($datetime) - ${SENSOR} - ${SENSOR} is reporting data again" >> ${logtemp}
+						rm -f ${TMPFOLDER}/${SENSOR}-error.reading
+					
+					fi					
+					temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
 	
-				echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
-				echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
-				echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
+					echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
+					echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
+					echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
 			
-				if [ "${last_temp[${SENSOR}]}" != "" ]; then
-					temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
-					[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
-				else
-					last_temp[${SENSOR}]=0
-					temp_diff=0
-				fi
-					
-				if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
-					touch ${TMPFOLDER}/temp.warn
-					if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
-						SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
-						echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
-						last_temp[${SENSOR}]=$temp_f
-						echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
-					fi
-				
-				elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
-					touch ${TMPFOLDER}/temp.crit
-					if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
-						SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
-Shutting down servers!!" 1
-						echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
-						echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
-						echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${logfile}
-						last_temp[${SENSOR}]=$temp_f
-						SHUTDOWN_MAIN ${SENSOR} &
-						echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
-
-					fi
-				elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
-					touch ${TMPFOLDER}/temp.critsys
-					if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
-						SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
-Shutting down system servers!!" 1
-						echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
-						echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
-						echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${logfile}
-						last_temp[${SENSOR}]=$temp_f
-						SHUTDOWN_SYS ${SENSOR} TEMP &
-						SHUTDOWN_CRIT ${SENSOR} &
-						echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
-					fi
-				else
-					[ ! -f ${TMPFOLDER}/temp.* ] && [ $(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/.${SENSOR}.sentdaily)) -gt 600 ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ] && touch ${TMPFOLDER}/.${SENSOR}.sentdaily && SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)" &
-					
-					if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
-						SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
-Previous Temp: ${last_temp[${SENSOR}]}'F"
-						echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
-						echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
-						echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${logfile}
-						[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
-						
-					elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
-						SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
-NORMAL TEMP: ${temp_f}'F"
-						echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
-						echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${logfile}
-						[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
-						
-					elif [ $relog -eq 1 ]; then
-						echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp}
-						
-					fi
-					rm -f ${TMPFOLDER}/temp.*
-					last_temp[${SENSOR}]=1
-				fi
-				
-				[ -f ${TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
-				if [ "$(compgen -G "${TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then
-					wait=60
-				else
-					wait=300
-				fi
-				if [ ${lastinsert} -ge ${wait} ]; then
-					if [ "${temp_h}" != "" ]; then
-						QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
+					if [ "${last_temp[${SENSOR}]}" != "" ]; then
+						temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
+						[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
 					else
-						QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
+						last_temp[${SENSOR}]=0
+						temp_diff=0
 					fi
-					${mysql_conn} -e "${QRY}"
-					touch ${TMPFOLDER}/${SENSOR}.insert
+					
+					if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
+						touch ${TMPFOLDER}/temp.warn
+						if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
+							SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
+							echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
+							last_temp[${SENSOR}]=$temp_f
+							echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
+						fi
+				
+					elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
+						touch ${TMPFOLDER}/temp.crit
+						if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
+							SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
+	Shutting down servers!!" 1
+							echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
+							echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
+							echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${logfile}
+							last_temp[${SENSOR}]=$temp_f
+							SHUTDOWN_MAIN ${SENSOR} &
+							echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
+
+						fi
+					elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
+						touch ${TMPFOLDER}/temp.critsys
+						if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
+							SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
+	Shutting down system servers!!" 1
+							echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
+							echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
+							echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${logfile}
+							last_temp[${SENSOR}]=$temp_f
+							SHUTDOWN_SYS ${SENSOR} TEMP &
+							SHUTDOWN_CRIT ${SENSOR} &
+							echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
+						fi
+					else
+						[ ! -f ${TMPFOLDER}/temp.* ] && [ $(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/.${SENSOR}.sentdaily)) -gt 600 ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ] && touch ${TMPFOLDER}/.${SENSOR}.sentdaily && SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)" &
+					
+						if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
+							SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
+	Previous Temp: ${last_temp[${SENSOR}]}'F"
+							echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
+							echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
+							echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${logfile}
+							[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
+						
+						elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
+							SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
+	NORMAL TEMP: ${temp_f}'F"
+							echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
+							echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${logfile}
+							[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
+						
+						elif [ $relog -eq 1 ]; then
+							echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp}
+						
+						fi
+						rm -f ${TMPFOLDER}/temp.*
+						last_temp[${SENSOR}]=1
+					fi
+				
+					[ -f ${TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
+					if [ "$(compgen -G "${TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then
+						wait=60
+					else
+						wait=300
+					fi
+					if [ ${lastinsert} -ge ${wait} ]; then
+						if [ "${temp_h}" != "" ]; then
+							QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
+						else
+							QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
+						fi
+						${mysql_conn} -e "${QRY}"
+						touch ${TMPFOLDER}/${SENSOR}.insert
+					fi
+				else
+					if [ ! -f ${TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
+						SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading sensor data" 1
+						echo "($datetime) - ${SENSOR} - ERROR reading sensor data" >> ${logtemp}
+						touch ${TMPFOLDER}/${SENSOR}-error.reading
+					fi
+
 				fi
+				
 			else
-				SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1
-				echo "($datetime) - alert sent" >> ${logtemp}
+				if [ ! -f ${TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
+					SENDNOTICE "${SENSOR} ERROR" "${SENSOR} is down" 1
+					echo "($datetime) - ${SENSOR} - ${SENSOR} is down" >> ${logtemp}
+					touch ${TMPFOLDER}/${SENSOR}.down
+				
+				fi				
 			fi
 			
 		done