update

2024-01-27 16:30:13 -06:00
parent 16a27daa4a
commit a940b1ae18
2 changed files with 49 additions and 42 deletions
--- a/inc/temp.inc.sh
+++ b/inc/temp.inc.sh
@@ -327,7 +327,7 @@ CHECKTEMP_SERVICE(){
 	DEFAULT_ESXI_FANSPEED=40
 	echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - 'Check Temp' Service Startup" >> ${PW_LOGFILE}
 	SENDTEMPS "TEMP Service Startup" &
-	[ ! -d ${PW_TMPFOLDER}/.lastfancheck ] && mkdir ${PW_TMPFOLDER}/.lastfancheck
+	[ ! -d ${PW_TMPFOLDER}/.lastfancheck ] && mkdir ${PW_TMPFOLDER}/.lastfancheck || rm -f ${PW_TMPFOLDER}/.lastfancheck/*.set
 	[ ! -d ${PW_TMPFOLDER}/.lastinsert ] && mkdir ${PW_TMPFOLDER}/.lastinsert
 	[ -d ${PW_TMPFOLDER}/.lasttemp ] && rm -Rf ${PW_TMPFOLDER}/.lasttemp
 	mkdir ${PW_TMPFOLDER}/.lasttemp
@@ -400,7 +400,7 @@ CHECKTEMP_SERVICE_RUNSENSOR(){
 			[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
 			t=1
 			
-			####### Fan Issue Workaround #######
+			####### Grab Fan Speeds #######
 			if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && ([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})) -ge 300 ]); then
 				# vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
 				vHOSTFAN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1)
@@ -544,8 +544,52 @@ CHECKTEMP_SERVICE_RUNSENSOR(){
 						echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
 						
 						if [[ "${vSENSOR}" = *"Exhaust"* ]]; then
-							if (( $(bc <<< "${temp_f} >= 100") )); then
-								temp=temp
+							if [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then
+								if [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set)) -ge 600 ]; then
+									if [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && (( $(bc <<< "${temp_f} < 100") )); then
+										if [ ${FanSpeed} -ne ${DEFAULT_ESXI_FANSPEED} ]; then
+											if [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then
+												SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FanSpeed Control" "Settings fans back to normal ${DEFAULT_ESXI_FANSPEED}%"
+												ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
+												ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x$(printf "%X" "${DEFAULT_ESXI_FANSPEED}") >/dev/null 2>&1
+												echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to ${DEFAULT_ESXI_FANSPEED}%" >> ${logtemp}
+												echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to ${DEFAULT_ESXI_FANSPEED}%" >> ${PW_LOGFILE}
+												echo "${DEFAULT_ESXI_FANSPEED}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set
+											elif [ ${FanSpeed} -ge 90 ]; then
+												SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FAN ISSUE?!" "Possible fan issue, currently running at ${FanSpeed}%" 1
+											fi
+										fi
+										([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ "$(cat ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})" != "${FanSpeed}" ]) && echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}
+										echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set
+																			
+									elif (( $(bc <<< "${temp_f} >= 100") )); then
+										[ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && NEW_FanSpeed=$(expr ${DEFAULT_ESXI_FANSPEED} + 5) || NEW_FanSpeed=$(expr ${FanSpeed} + 5)
+										if [ ${NEW_FanSpeed} -le 104 ]; then
+											[ ${NEW_FanSpeed} -gt 100 ] && NEW_FanSpeed=100
+											SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Exhaust Warm" "Increasing fans to ${NEW_FanSpeed}%"
+											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
+											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x$(printf "%X" "${NEW_FanSpeed}") >/dev/null 2>&1
+											echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Increasing fans to ${NEW_FanSpeed}%" >> ${logtemp}
+											echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Increasing fans to ${NEW_FanSpeed}%" >> ${PW_LOGFILE}
+											echo "${NEW_FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set
+										fi
+										
+									else
+										NEW_FanSpeed=$(expr ${FanSpeed} - 5)
+										if [ ${NEW_FanSpeed} -ge $(expr ${DEFAULT_ESXI_FANSPEED} - 4) ]; then
+											[ ${NEW_FanSpeed} -lt ${DEFAULT_ESXI_FANSPEED} ] && NEW_FanSpeed=${DEFAULT_ESXI_FANSPEED}
+											SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Exhaust Cooling" "Decreasing fans to ${NEW_FanSpeed}%"
+											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
+											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x$(printf "%X" "${NEW_FanSpeed}") >/dev/null 2>&1
+											echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Decreasing fans to ${NEW_FanSpeed}%" >> ${logtemp}
+											echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Decreasing fans to ${NEW_FanSpeed}%" >> ${PW_LOGFILE}
+											echo "${NEW_FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set
+										fi
+										
+									fi
+										
+									[ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set
+								fi
 							fi
 						fi
 						
@@ -577,25 +621,6 @@ Previous Temp: ${last_temp}${GAUGESH}"
 							echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR}
 						fi
 				
-						####### Fan Issue Workaround #######
-						if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ "${FanSpeed}" != "" ]; then
-							if [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})) -ge 300 ]; then
-								if [ ${FanSpeed} -ne ${DEFAULT_ESXI_FANSPEED} ]; then
-									
-									if [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then
-										SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal ${DEFAULT_ESXI_FANSPEED}%"
-										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
-										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x$(printf "%X" "${DEFAULT_ESXI_FANSPEED}") >/dev/null 2>&1
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to ${DEFAULT_ESXI_FANSPEED}%" >> ${logtemp}
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to ${DEFAULT_ESXI_FANSPEED}%" >> ${PW_LOGFILE}
-									elif [ ${FanSpeed} -ge 90 ]; then
-										SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FAN ISSUE?!" "Possible fan issue, currently running at ${FanSpeed}%" 1
-									fi
-								fi
-								([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ "$(cat ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})" != "${FanSpeed}" ]) && echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}
-							fi
-						fi
-				
 					elif [[ "${SENSORa}" != *"-Usage"* ]] && [[ "${SENSORa}" != *"FAN"* ]]; then
 						
 						if ([ ! "${last_temp}" ] || (( $(bc <<< "${last_temp} < 2") ))); then
@@ -621,24 +646,6 @@ Previous Temp: ${last_temp}${GAUGESH}"
 						
 							fi
 							
-							####### Fan Issue Workaround #######
-							if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ "${FanSpeed}" != "" ]]; then
-								if [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})) -ge 300 ]; then
-									# if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then
-									if [ ${FanSpeed} -ne 80 ]; then
-										if [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then
-											SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%" 1
-											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
-											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
-											echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${logtemp}
-											echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${PW_LOGFILE}
-										else
-											SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Fans not set at 80%" 1
-										fi
-									fi
-									([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ "$(cat ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})" != "${FanSpeed}" ]) && echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}
-								fi
-							fi
 												
 						elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then							
 							if (( $(bc <<< "${temp_diff} >= 1") )); then