From f4320f6cce4f2d35feb7680ba6f8e19bf757a1f3 Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Tue, 26 Dec 2023 19:18:15 -0600
Subject: [PATCH] update

---
 inc/temp.inc.sh | 604 +++++++++++++++++++++++++-----------------------
 powerwall.sh    |   4 +-
 2 files changed, 315 insertions(+), 293 deletions(-)

diff --git a/inc/temp.inc.sh b/inc/temp.inc.sh
index 3a7f8cba..91cfd158 100644
--- a/inc/temp.inc.sh
+++ b/inc/temp.inc.sh
@@ -307,9 +307,9 @@ CHECKTEMP_SERVICE(){
 		# 		touch ${PW_TMPFOLDER}/.sentdaily
 		# 		SENDDAILY &
 		# 	fi
-		# fi	
+		# fi
+		SENSORS_CHECKED=()
 		for SENSORa in ${PW_SENSOR_ORD[@]}; do
-			echo -en "${SENSORa}"; start_sub=`date +%s`
 			logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa}
 			[ ! -f ${logtemp} ] && touch ${logtemp}
 			[ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && relog=1 || relog=0
@@ -321,295 +321,9 @@ CHECKTEMP_SERVICE(){
 					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp}
 					rm -f ${PW_TMPFOLDER}/${SENSORa}.down
 				fi
-				if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
-					vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
-					IFS=$'\n'
-					read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
-					unset IFS
-					[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
-					t=1
-					
-				else
-					SENSORS_CHECK=(${SENSORa})
-				fi
-				if [ "${SENSORS_CHECK}" != "" ]; then
-					for SENSOR in "${SENSORS_CHECK[@]}"; do
-						if [ "${PW_THRESHOLDS[${SENSORa}]}" != "" ]; then
-							echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 1 | read temp_warn
-							echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 2 | read temp_crit	
-							[ "$(echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3)" != "" ] && echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3 | read temp_crit_sys || temp_crit_sys=${temp_crit}
-						fi
+				CHECKTEMP_SERVICE_RUNSENSOR ${SENSORa} &
+				SENSORS_CHECKED+=(${SENSORa})
 				
-						if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
-							mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
-							echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
-							echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
-							
-						elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then
-							INFO=$(curl -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php | jq)
-							if [[ "${SENSOR}" = *"-Probe" ]]; then
-								temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes`
-							else
-								temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes`
-							fi
-							temp_h=$(echo ${INFO} | jq -r '.device0 .obs[].humidity')
-								
-						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
-							if [[ "${SENSOR}" = *"CPU"* ]]; then
-								temp_c=$(vcgencmd measure_temp)
-								temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
-								temp_h=''
-							elif [[ "${SENSOR}" = *"FAN"* ]]; then
-								temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input)
-								temp_h=''
-							fi
-		
-						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
-							temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
-							temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
-							temp_h=''
-				
-						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
-							temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
-							temp_c=`echo "scale=1; ${temp_c}/10" | bc`
-							temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
-							temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
-							temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc)
-							temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
-							temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
-							temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc)
-							temp_crit_sys=${temp_crit}
-							temp_h=''
-							vSENSOR=${SENSOR}
-							SENSOR="${SENSORa}-${vSENSOR// /_}"
-							((t++))
-							
-						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
-							if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then
-								temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
-							else
-								temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage")
-							fi
-							temp_h=''
-							
-						fi
-						[ "${last_temp[${SENSOR//-/}]}" == "" ] && last_temp[${SENSOR//-/}]=0
-						
-						logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
-						if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
-							if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
-								GAUGE="CPU-USAGE"
-							elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
-								GAUGE="RPM"
-							else
-								GAUGE="TEMP"
-							fi
-							if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
-								GAUGESH='%'
-							elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
-								GAUGESH="'R"
-							else
-								GAUGESH="'F"
-							fi
-							[ -f ${PW_TMPFOLDER}/${SENSOR}.* ] && rm -f ${PW_TMPFOLDER}/${SENSOR}.*
-							if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ]; then
-								SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
-								echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
-								rm -f ${PW_TMPFOLDER}/${SENSOR}-error.*
-			
-							fi
-							
-							if [ "${SENSOR}" == "ServerRoomTH" ]; then
-								temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc`
-							else
-								if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ]; then
-									temp_f=$(echo "scale=2; $temp_c/1" | bc -l)
-								elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then
-									temp_f=${temp_c}
-								else
-									temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32")
-								fi
-								(( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}"
-							fi
-							
-							# echo "HERE: [${SENSOR//-/}] - ${last_temp[${SENSOR//-/}]} - ${temp_c} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
-							# [ ! "${temp_c}" ] || [ ! "${temp_f}" ] || [ ! "${temp_warn}" ] || [ ! "${temp_crit}" ] || [ ! "${temp_crit_sys}" ] && echo "VALUE NOT FOUND"
-					
-							if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
-								echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
-								echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
-								echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
-							fi
-							
-							# iDS-vMS-Offsite Fan Issue Workaround
-							if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
-								vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
-								# vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
-								if [ "${vHOSTFAN}" != "" ]; then
-									FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=$(ROUND_NUMBER `echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`)
-									# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
-								else
-									FanSpeed=''
-								fi
-							fi
-							
-							if (( $(bc <<< "${temp_f} < ${temp_warn}") )); then
-								if (( $(bc <<< "${last_temp[${SENSOR//-/}]} > 1") )); then
-									[ -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ] && SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH}
-Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
-									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR//-/}]}${GAUGESH}" >> ${logtemp}
-									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE}
-									# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
-				
-								elif [ "${last_temp[${SENSOR//-/}]}" == "0" ]; then
-									# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
-# NORMAL TEMP: ${temp_f}${GAUGESH}"
-									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp}
-									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
-									# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
-				
-								elif [ "${relog}" == "1" ]; then
-									echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp}
-				
-								fi
-								rm -f ${PW_TMPFOLDER}/temp.*
-								rm -f ${PW_TMPFOLDER}/power.${SENSORa}.*
-								last_temp[${SENSOR//-/}]=1
-								
-								# iDS-vMS-Offsite Fan Issue Workaround
-								if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${FanSpeed}" != "" ]; then
-									if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then
-									# if [ ${FanSpeed} -ne 30 ]; then
-										SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
-										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
-										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${logtemp}
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${PW_LOGFILE}
-									fi
-								fi
-								
-							else
-								([ ! "${last_temp[${SENSOR//-/}]}" ] || (( $(bc <<< "${last_temp[${SENSOR//-/}]} < 2") ))) && last_temp[${SENSOR//-/}]=${temp_f}
-								temp_diff=`echo "scale=2; ${last_temp[${SENSOR//-/}]}-${temp_f}" | bc`
-								[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`								
-								
-								if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
-									touch ${PW_TMPFOLDER}/temp.warn
-									if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
-										SENDNOTICE "${SENSOR} ${GAUGE} WARNING" "WARNING ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
-										last_temp[${SENSOR//-/}]=${temp_f}
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - WARNING ${GAUGE} - (Difference of ${temp_diff}')" >> ${logtemp}
-										
-									fi
-									
-									# iDS-vMS-Offsite Fan Issue Workaround
-									if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
-										SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
-										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
-										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${logtemp}
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${PW_LOGFILE}
-									fi
-									
-								elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then
-									if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
-										SENDNOTICE "${SENSOR} ${GAUGE} CRITICAL" "CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - CRITICAL - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp}
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
-										last_temp[${SENSOR//-/}]=${temp_f}
-									
-										if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f touch ${PW_TMPFOLDER}/temp.crit ]; then
-											touch ${PW_TMPFOLDER}/temp.crit
-											SHUTDOWN_MAIN ${SENSOR} &
-											echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp}
-										
-										# elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
-										# 	SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
-										# 	SHUTDOWN_SERVER ${SENSORa} -esxi off &
-										# 	touch ${PW_TMPFOLDER}/power.${SENSORa}.off
-
-										fi
-									fi
-								elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then
-									if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
-										SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
-										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
-										last_temp[${SENSOR//-/}]=${temp_f}
-										if [ ! -f touch ${PW_TMPFOLDER}/temp.critsys ]; then
-											touch ${PW_TMPFOLDER}/temp.critsys
-											[ "${SENSOR}" == "ServerRoomTH" ] && SHUTDOWN_SYS ${SENSOR} TEMP &
-											SHUTDOWN_CRIT ${SENSOR} &
-											echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
-										fi
-									fi
-								fi
-							fi
-							
-							[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
-							if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
-								wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}*60" | bc)
-							else
-								wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_NORMAL}*60" | bc)
-							fi
-							
-							# [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ] && temp_probe_f=$(bc <<< "scale=2; ${temp_probe_c}*1.8+32")
-							
-							if [ "${PW_SENSOR_ID[${SENSORa}]}" != "" ] && [ ${lastinsert} -ge ${wait} ]; then
-								if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
-									entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
-									if [ "${entityid[1]}" == "" ]; then
-										${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
-										entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
-									fi
-									entityid=${entityid[1]}
-									
-									QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
-									if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ] && [ "${vHOSTFAN}" != "" ]; then
-										${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')"
-									fi
-								elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
-									${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
-								elif [ "${temp_h}" != "" ]; then
-									QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')"
-								else
-									QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
-								fi
-								${mysql_conn} -e "${QRY}"
-								# if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then
-								# 	${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID["${SENSORa}-Probe"]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_probe_f}')"
-								# fi
-								touch ${PW_TMPFOLDER}/${SENSOR}.insert
-							fi
-							
-							#### LOGGING OF SERVERROOM SENSOR DIFFERENCE
-							if [ "${SENSOR}" == "ServerRoomLA-Probe" ]; then
-								if [ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.log.sr.difference)) -ge 300 ]; then
-									if [ "${temp_f}" != "" ] && [ "${ServerRoomTH}" != "" ]; then
-										[ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] && touch ${PW_TMPFOLDER}/.log.sr.difference
-										temp_difference=$(bc <<< "scale=2; ${ServerRoomTH}-${temp_f}")
-										# (( $(bc <<<"${temp_difference} < 0") )) && temp_difference=`echo "scale=2; ${temp_difference} * -1" | bc`
-										echo "$(date "+%Y-%m-%d %H:%M:%S")~${temp_difference}" >> ${PW_TMPFOLDER}/.log.sr.difference
-									fi
-								fi
-							elif [ "${SENSOR}" == "ServerRoomTH" ]; then
-								export ServerRoomTH=${temp_f}
-							fi					
-							###################
-							
-						else
-							[ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] && touch ${PW_TMPFOLDER}/${SENSOR}.down
-							if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
-								touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
-							elif [ ! -f ${PW_TMPFOLDER}/${SENSOR}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then
-								touch ${PW_TMPFOLDER}/${SENSOR}-error.sent
-								SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
-								echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
-							fi	
-						fi
-					done
-				fi
-				# elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down
 			else
 				[ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && touch ${PW_TMPFOLDER}/${SENSORa}.down
 				if [ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.reading ]; then
@@ -620,12 +334,320 @@ Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
 					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} Sensor - ERROR reading sensor data" >> ${logtemp}
 				fi	
 			fi
-			end_sub=`date +%s`; runtime_sub=$((end_sub-start_sub)); echo " - done: ${runtime_sub}s"
 		done
 		
+		for SENSORa in "${SENSORS_CHECKED[@]}"; do
+			[ ! -f ${dli//.down/.disable} ] && SENSORS_CHECKED+=(${SENSORa})
+		done
+		
+		s_count=0; completed=false
+		until [ "${completed}" == "true" ]; do
+			for sensor in ${PW_TMPFOLDER}/zservicerun.*.done; do
+				SENSORa=$(grep -oP '(?<=zservicerun.).*?(?=.done)' <<< "${nodestatus}")
+				if [ "${SENSORa}" != "*" ]; then
+					rm -f ${sensor}
+					((s_count++))
+					# echo "${NTS} == ${s_count} == ${nodestatus}"
+				fi
+			done
+			[ ${s_count} -eq ${#SENSORS_CHECKED[@]} ] && completed=true
+		done		
 		end=`date +%s`; runtime=$((end-start)); echo "Total runtime: ${runtime}s"
 		# sleep 10s
 	done # &
+	${PW_TMPFOLDER}/zservicerun.${SENSORa}.done
+}
+CHECKTEMP_SERVICE_RUNSENSOR(){
+	SENSORa=${1}
+	if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
+		vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
+		IFS=$'\n'
+		read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
+		unset IFS
+		[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
+		t=1
+	
+	else
+		SENSORS_CHECK=(${SENSORa})
+	fi
+	if [ "${SENSORS_CHECK}" != "" ]; then
+		for SENSOR in "${SENSORS_CHECK[@]}"; do
+			if [ "${PW_THRESHOLDS[${SENSORa}]}" != "" ]; then
+				echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 1 | read temp_warn
+				echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 2 | read temp_crit	
+				[ "$(echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3)" != "" ] && echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3 | read temp_crit_sys || temp_crit_sys=${temp_crit}
+			fi
+
+			if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
+				mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
+				echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
+				echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
+			
+			elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then
+				INFO=$(curl -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php | jq)
+				if [[ "${SENSOR}" = *"-Probe" ]]; then
+					temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes`
+				else
+					temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes`
+				fi
+				temp_h=$(echo ${INFO} | jq -r '.device0 .obs[].humidity')
+				
+			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
+				if [[ "${SENSOR}" = *"CPU"* ]]; then
+					temp_c=$(vcgencmd measure_temp)
+					temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
+					temp_h=''
+				elif [[ "${SENSOR}" = *"FAN"* ]]; then
+					temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input)
+					temp_h=''
+				fi
+
+			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
+				temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
+				temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
+				temp_h=''
+
+			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
+				temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
+				temp_c=`echo "scale=1; ${temp_c}/10" | bc`
+				temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
+				temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
+				temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc)
+				temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
+				temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
+				temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc)
+				temp_crit_sys=${temp_crit}
+				temp_h=''
+				vSENSOR=${SENSOR}
+				SENSOR="${SENSORa}-${vSENSOR// /_}"
+				((t++))
+			
+			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
+				if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then
+					temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
+				else
+					temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage")
+				fi
+				temp_h=''
+			
+			fi
+			[ "${last_temp[${SENSOR//-/}]}" == "" ] && last_temp[${SENSOR//-/}]=0
+		
+			logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
+			if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
+				if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
+					GAUGE="CPU-USAGE"
+				elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
+					GAUGE="RPM"
+				else
+					GAUGE="TEMP"
+				fi
+				if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
+					GAUGESH='%'
+				elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
+					GAUGESH="'R"
+				else
+					GAUGESH="'F"
+				fi
+				[ -f ${PW_TMPFOLDER}/${SENSOR}.* ] && rm -f ${PW_TMPFOLDER}/${SENSOR}.*
+				if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ]; then
+					SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
+					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
+					rm -f ${PW_TMPFOLDER}/${SENSOR}-error.*
+
+				fi
+			
+				if [ "${SENSOR}" == "ServerRoomTH" ]; then
+					temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc`
+				else
+					if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ]; then
+						temp_f=$(echo "scale=2; $temp_c/1" | bc -l)
+					elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then
+						temp_f=${temp_c}
+					else
+						temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32")
+					fi
+					(( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}"
+				fi
+			
+				# echo "HERE: [${SENSOR//-/}] - ${last_temp[${SENSOR//-/}]} - ${temp_c} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
+				# [ ! "${temp_c}" ] || [ ! "${temp_f}" ] || [ ! "${temp_warn}" ] || [ ! "${temp_crit}" ] || [ ! "${temp_crit_sys}" ] && echo "VALUE NOT FOUND"
+	
+				if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
+					echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
+					echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
+					echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
+				fi
+			
+				# iDS-vMS-Offsite Fan Issue Workaround
+				if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
+					vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
+					# vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
+					if [ "${vHOSTFAN}" != "" ]; then
+						FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=$(ROUND_NUMBER `echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`)
+						# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
+					else
+						FanSpeed=''
+					fi
+				fi
+			
+				if (( $(bc <<< "${temp_f} < ${temp_warn}") )); then
+					if (( $(bc <<< "${last_temp[${SENSOR//-/}]} > 1") )); then
+						[ -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ] && SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH}
+	Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
+						echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR//-/}]}${GAUGESH}" >> ${logtemp}
+						echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE}
+						# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
+
+					elif [ "${last_temp[${SENSOR//-/}]}" == "0" ]; then
+						# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
+	# NORMAL TEMP: ${temp_f}${GAUGESH}"
+						echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp}
+						echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
+						# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
+
+					elif [ "${relog}" == "1" ]; then
+						echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp}
+
+					fi
+					rm -f ${PW_TMPFOLDER}/temp.*
+					rm -f ${PW_TMPFOLDER}/power.${SENSORa}.*
+					last_temp[${SENSOR//-/}]=1
+				
+					# iDS-vMS-Offsite Fan Issue Workaround
+					if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${FanSpeed}" != "" ]; then
+						if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then
+						# if [ ${FanSpeed} -ne 30 ]; then
+							SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
+							ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
+							ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
+							echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${logtemp}
+							echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${PW_LOGFILE}
+						fi
+					fi
+				
+				else
+					([ ! "${last_temp[${SENSOR//-/}]}" ] || (( $(bc <<< "${last_temp[${SENSOR//-/}]} < 2") ))) && last_temp[${SENSOR//-/}]=${temp_f}
+					temp_diff=`echo "scale=2; ${last_temp[${SENSOR//-/}]}-${temp_f}" | bc`
+					[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`								
+				
+					if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
+						touch ${PW_TMPFOLDER}/temp.warn
+						if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
+							SENDNOTICE "${SENSOR} ${GAUGE} WARNING" "WARNING ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
+							last_temp[${SENSOR//-/}]=${temp_f}
+							echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - WARNING ${GAUGE} - (Difference of ${temp_diff}')" >> ${logtemp}
+						
+						fi
+					
+						# iDS-vMS-Offsite Fan Issue Workaround
+						if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
+							SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
+							ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
+							ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
+							echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${logtemp}
+							echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${PW_LOGFILE}
+						fi
+					
+					elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then
+						if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
+							SENDNOTICE "${SENSOR} ${GAUGE} CRITICAL" "CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
+							echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - CRITICAL - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp}
+							echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
+							last_temp[${SENSOR//-/}]=${temp_f}
+					
+							if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f touch ${PW_TMPFOLDER}/temp.crit ]; then
+								touch ${PW_TMPFOLDER}/temp.crit
+								SHUTDOWN_MAIN ${SENSOR} &
+								echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp}
+						
+							# elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
+							# 	SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
+							# 	SHUTDOWN_SERVER ${SENSORa} -esxi off &
+							# 	touch ${PW_TMPFOLDER}/power.${SENSORa}.off
+
+							fi
+						fi
+					elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then
+						if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
+							SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
+							echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
+							echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
+							last_temp[${SENSOR//-/}]=${temp_f}
+							if [ ! -f touch ${PW_TMPFOLDER}/temp.critsys ]; then
+								touch ${PW_TMPFOLDER}/temp.critsys
+								[ "${SENSOR}" == "ServerRoomTH" ] && SHUTDOWN_SYS ${SENSOR} TEMP &
+								SHUTDOWN_CRIT ${SENSOR} &
+								echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
+							fi
+						fi
+					fi
+				fi
+			
+				[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
+				if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
+					wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}*60" | bc)
+				else
+					wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_NORMAL}*60" | bc)
+				fi
+			
+				# [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ] && temp_probe_f=$(bc <<< "scale=2; ${temp_probe_c}*1.8+32")
+			
+				if [ "${PW_SENSOR_ID[${SENSORa}]}" != "" ] && [ ${lastinsert} -ge ${wait} ]; then
+					if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
+						entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
+						if [ "${entityid[1]}" == "" ]; then
+							${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
+							entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
+						fi
+						entityid=${entityid[1]}
+					
+						QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
+						if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ] && [ "${vHOSTFAN}" != "" ]; then
+							${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')"
+						fi
+					elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
+						${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
+					elif [ "${temp_h}" != "" ]; then
+						QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')"
+					else
+						QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
+					fi
+					${mysql_conn} -e "${QRY}"
+					# if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then
+					# 	${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID["${SENSORa}-Probe"]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_probe_f}')"
+					# fi
+					touch ${PW_TMPFOLDER}/${SENSOR}.insert
+				fi
+			
+				#### LOGGING OF SERVERROOM SENSOR DIFFERENCE
+				if [ "${SENSOR}" == "ServerRoomLA-Probe" ]; then
+					if [ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.log.sr.difference)) -ge 300 ]; then
+						if [ "${temp_f}" != "" ] && [ "${ServerRoomTH}" != "" ]; then
+							[ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] && touch ${PW_TMPFOLDER}/.log.sr.difference
+							temp_difference=$(bc <<< "scale=2; ${ServerRoomTH}-${temp_f}")
+							# (( $(bc <<<"${temp_difference} < 0") )) && temp_difference=`echo "scale=2; ${temp_difference} * -1" | bc`
+							echo "$(date "+%Y-%m-%d %H:%M:%S")~${temp_difference}" >> ${PW_TMPFOLDER}/.log.sr.difference
+						fi
+					fi
+				elif [ "${SENSOR}" == "ServerRoomTH" ]; then
+					export ServerRoomTH=${temp_f}
+				fi					
+				###################
+			
+			else
+				[ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] && touch ${PW_TMPFOLDER}/${SENSOR}.down
+				if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
+					touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
+				elif [ ! -f ${PW_TMPFOLDER}/${SENSOR}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then
+					touch ${PW_TMPFOLDER}/${SENSOR}-error.sent
+					SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
+					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
+				fi	
+			fi
+		done
+	fi
+	touch ${PW_TMPFOLDER}/zservicerun.${SENSORa}.done
 }
 
 DAILYTEMP(){
diff --git a/powerwall.sh b/powerwall.sh
index 95338ab3..5485875d 100755
--- a/powerwall.sh
+++ b/powerwall.sh
@@ -79,7 +79,7 @@ CHECK_SERVICES(){
 }
 
 SHUTDOWN_MAIN(){
-	if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "ServerRoomLA" ] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
+	if [ "${1}" == "ServerRoomTH" ] || [[ "${1}" = *"ServerRoomLA"* ]] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
 		touch ${PW_TMPFOLDER}/shutdown.MAIN.started
 		SHUTDOWN_SERVER main -esxi off
 		[ "${1}" == "POWER" ] && reason='Low Power!' || reason='Server Room Overheated!'
@@ -98,7 +98,7 @@ ${reason}" 1
 	echo
 }
 SHUTDOWN_SYS(){
-	if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "ServerRoomLA" ] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
+	if [ "${1}" == "ServerRoomTH" ] || [[ "${1}" = *"ServerRoomLA"* ]] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
 		if [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ]; then
 			touch ${PW_TMPFOLDER}/shutdown.MAINSITE.started
 			SHUTDOWN_SERVER mainsite -esxi off