From f3832a62c10d39835477929aca6753c58d5139db Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Sun, 17 Dec 2023 13:07:19 -0600
Subject: [PATCH] update

---
 defaults.inc    |  2 +-
 inc/temp.inc.sh | 59 ++++++++++++++++++++++++++++++-------------------
 2 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/defaults.inc b/defaults.inc
index 6bd7c64c..67d71aea 100755
--- a/defaults.inc
+++ b/defaults.inc
@@ -1,5 +1,5 @@
 
-VERS='2.5.34-12172023'
+VERS='2.5.35-12172023'
 
 noheader=' update service dailytemp confsync '
 
diff --git a/inc/temp.inc.sh b/inc/temp.inc.sh
index 9c36996f..d659a7fd 100644
--- a/inc/temp.inc.sh
+++ b/inc/temp.inc.sh
@@ -329,10 +329,10 @@ CHECKTEMP_SERVICE(){
 						if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
 							[ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ] && GAUGE="CPU-USAGE" || GAUGE="TEMP"
 							[ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ] && GAUGESH='%' || GAUGESH="'F"
-							if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
+							if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ]; then
 								SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
 								echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
-								rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
+								rm -f ${PW_TMPFOLDER}/${SENSOR}-error.*
 			
 							fi
 							
@@ -352,6 +352,18 @@ CHECKTEMP_SERVICE(){
 								echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
 							fi
 							
+							# iDS-vMS-Offsite Fan Issue Workaround
+							if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
+								vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
+								# vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
+								if [ "${vHOSTFAN}" != "" ]; then
+									FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
+									# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
+								else
+									FanSpeed=''
+								fi
+							fi
+							
 							if (( $(bc <<< "${temp_f} < ${temp_warn}") )); then
 								if (( $(bc <<< "${last_temp[${SENSOR//-/}]} > 1") )); then
 									SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH}
@@ -376,18 +388,14 @@ Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
 								last_temp[${SENSOR//-/}]=1
 								
 								# iDS-vMS-Offsite Fan Issue Workaround
-								if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
-									vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
-									# vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
-									if [ "${vHOSTFAN}" != "" ]; then
-										FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
-										# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
-										if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then
-										# if [ ${FanSpeed} -ne 30 ]; then
-											SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
-											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
-											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
-										fi
+								if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${FanSpeed}" != "" ]; then
+									if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then
+									# if [ ${FanSpeed} -ne 30 ]; then
+										SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
+										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
+										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
+										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${logtemp}
+										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${PW_LOGFILE}
 									fi
 								fi
 								
@@ -402,15 +410,18 @@ Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
 										SENDNOTICE "${SENSOR} ${GAUGE} WARNING" "WARNING ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
 										last_temp[${SENSOR//-/}]=${temp_f}
 										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - WARNING ${GAUGE} - (Difference of ${temp_diff}')" >> ${logtemp}
-							
-										# iDS-vMS-Offsite Fan Issue Workaround
-										if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
-											SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
-											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
-											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
-										fi							
+										
 									fi
-		
+									
+									# iDS-vMS-Offsite Fan Issue Workaround
+									if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
+										SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
+										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
+										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
+										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${logtemp}
+										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${PW_LOGFILE}
+									fi
+									
 								elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then
 									if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
 										SENDNOTICE "${SENSOR} ${GAUGE} CRITICAL" "CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
@@ -496,7 +507,9 @@ Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
 							###################
 							
 						else
-							if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
+							if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
+								touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
+							elif ([ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ]); then
 								SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
 								echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
 								touch ${PW_TMPFOLDER}/${SENSOR}-error.reading