CHECKTEMP(){
	start=`date +%s`
	cw=25;
	while [ $# -gt 0 ]; do
		case "$1" in
			-a|-avg) avgdays=${2};;
			-S|-sensors) temptype="sensors";;
			-e|-esxi) temptype="esxi";;
			-s|-search) search=${2};;
		esac
		shift 1
	done
	echo
	if [ "${avgdays}" != "" ]; then
		[ ${avgdays} -lt 10 ] && avgdays_disp=" ${avgdays}" || avgdays_disp=${avgdays}
		echo -e "${idsCL[Cyan]}Sensor Temperature(s)               Current / ${idsCL[LightYellow]}${avgdays_disp}DayAVG   ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}"
	else
		avgdays=noavg
		echo -e "${idsCL[Cyan]}Sensor Temperature(s)               Current    ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}"
	fi
	DIVIDER . yellow 90
		
	for SENSOR in ${PW_SENSOR_ORD[@]}; do
		
		if [ "${search}" == "" ] || [[ "${SENSOR,,}" = *"${search,,}"* ]]; then
			
			if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "esxi" ] && [ "${temptype}" != "sensors" ]; then
				vHOSTiDRACIP=${PW_HOST_IDRACS[${SENSOR}]}
				vHOSTIP=${PW_HOST_ADDRESSES[${SENSOR}]}				
				vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTiDRACIP}]}
				echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[White]}[${vHOSTiDRACIP}]${idsCL[Default]}"
				echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... "
				
				if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then
					vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
					IFS=$'\n'
					read -rd '' -a vHOST_TEMPNAMES <<<"${vHOST_TEMPNAMES}"
					unset IFS
					echo -en "\r\033[K"
					if [ ${#vHOST_TEMPNAMES[@]} > 1 ]; then
						t=1
						for sensorname in "${vHOST_TEMPNAMES[@]}"; do
							temp_c=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
							temp_c=`echo "scale=1; ${temp_c}/10" | bc`
							temp_warn=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
							temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
							temp_crit=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
							temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
							
							c=0; spc=''; spc1=`expr ${cw} - ${#sensorname} - 1`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
							
							[ "${avgdays}" != "noavg" ] && echo -en "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${idsCL[Yellow]}Calculating ${avgdays} Day average ... "
							reading=$(CHECKTEMPSENSOR ${SENSOR} ${temp_c} ${temp_warn} ${temp_crit} ${sensorname// /_} ${avgdays})
							[ "${avgdays}" != "noavg" ] && echo -en "\r\033[K"
							echo -e "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${reading}"
							
							((t++))
						done
					else
						echo -e "${idsCL[LightCyan]} ${vHOSTNAME}${spc}${idsCL[Default]}: No Data"
					fi
					
				else #host down
					echo -e "\r\033[K${idsCL[LightRed]}${SENSOR} Host is down"
					
					# if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
					# 	SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1
					# 	echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
					# 	touch ${PW_TMPFOLDER}/${EHOST}.down
					# fi
				fi
				echo
				
			elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" != "esxi" ] && [ "${temptype}" != "esxi" ]; then
				[[ "${SENSOR}" == *"CPU-Usage"* ]] && echo -en "\033[1A"
				c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
				echo -en "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${idsCL[Yellow]}"
				[ "${avgdays}" != "noavg" ] && echo -en "Pulling data and calculating ${avgdays} Day average ... " || echo -en "Pulling data ... "
				reading=$(CHECKTEMPSENSOR ${SENSOR} . . . . ${avgdays})
				echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${reading}"
				echo
			fi
		fi
	done
	end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
	[ "${PW_ACTION}" != "" ] && echo
}

CHECKTEMPSENSOR(){
	echo ${PW_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
	echo ${PW_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
	[ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ] && GAUGESH="'%" || GAUGESH="'F"
	
	if [ "${6}" != "noavg" ]; then
		[ "${6}" != "" ] && [[ "${6}" =~ ^[0-9]+$ ]] && avgdays=${6} || avgdays=3
		if [ "${PW_SENSOR_TYPE[${1}]}" != "esxi" ]; then
			average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays})
		else
			average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays} ${5})
		fi
	else
		average=''
	fi
		
	if [ "${PW_SENSOR_TYPE[${1}]}" == "mqtt" ]; then
		mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
		echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
		
	elif [ "${PW_SENSOR_TYPE[${1}]}" == "lacrosse" ]; then
		temp_c=$(GET_LACROSSE_INFO "${1}")
		
	elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ]; then
		temp_c=$(vcgencmd measure_temp)
		temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
		
	elif [ "${PW_SENSOR_TYPE[${1}]}" == "remotesystem" ]; then
		temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
		if [ "${temp_c}" != "" ]; then
			temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
			
		fi
	elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then
		temp_c=${2}
		temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'`
		temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'`
		
	elif [ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ]; then
		if [ "${1}" == "Powerwall-CPU-Usage" ]; then
			temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
		else
			temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} "/opt/idssys/defaults/get-data.sh cpu-usage")
		fi
	fi
	
	if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
		if [ "${1}" == "ServerRoomTH" ]; then
			temp_f=`echo "scale=1; $(bc <<< "scale=1; ${temp_c}*1.8+32")+${PW_ServerRoomTH_Dev}" | bc`
		elif [ "${1}" == "ServerRoomLA" ]; then
			temp_f=${temp_c}
		else
			[ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ] && temp_f=$(echo "scale=2; $temp_c/1" | bc -l) || temp_f=$(bc <<< "scale=1; ${temp_c}*1.8+32")
			(( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}"
		fi
		# echo "HERE: '${temp_c}' -> '${temp_f}'"
		# echo -en ""
		c=0; spct=''; spc1=`expr 6 - ${#temp_f}`; until [ ${c} = ${spc1} ]; do spct="${spct} "; c=`expr ${c} + 1`; done
		
		if [ "${average}" != "" ]; then
			c=0; spca=''; spc1=`expr 6 - ${#average}`; until [ ${c} = ${spc1} ]; do spca="${spca} "; c=`expr ${c} + 1`; done
			if [ "${average}" == "" ]; then
				averagedisp="${idsCL[Green]}"
			elif [ $(bc -l <<< "${average} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${average} < ${temp_crit}") -eq 1 ]; then
				averagedisp="${idsCL[Yellow]}${average}"
			elif [ $(bc -l <<< "${average} >= ${temp_crit}") -eq 1 ]; then
				averagedisp="${idsCL[LightRed]}${average}"
			else
				averagedisp="${idsCL[Green]}${average}"
			fi
		fi
		
		if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
			[ "${average}" != "" ] && echo -en "${idsCL[Yellow]}WARNING ${spct}${temp_f}${GAUGESH} /${spca}${averagedisp}${GAUGESH}${idsCL[Default]}" || echo -en "${idsCL[Yellow]}WARNING ${spct}${temp_f}${GAUGESH}${idsCL[Default]}"
		elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ]; then
			[ "${average}" != "" ] && echo -en "${idsCL[LightRed]}CRITICAL${spct}${temp_f}${GAUGESH} /${spca}${averagedisp}${GAUGESH}${idsCL[Default]}" || echo -en "${idsCL[LightRed]}CRITICAL${spct}${temp_f}${GAUGESH}${idsCL[Default]}"
		else
			[ "${average}" != "" ] && echo -en "${idsCL[Green]}Normal  ${spct}${temp_f}${GAUGESH} /${spca}${averagedisp}${GAUGESH}${idsCL[Default]}" || echo -en "${idsCL[Green]}Normal  ${spct}${temp_f}${GAUGESH}${idsCL[Default]}"
		fi
		
		tmp="${spct}${temp_f}"
		c=0; spc=''; spc1=`expr 10 - ${#tmp}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
		[ ${#temp_warn} == 2 ] && spcw='  ' || spcw=' '
		[ ${#temp_crit} == 2 ] && spcc='  ' || spcc=' '
		echo -e "${spc}[${spcw}${idsCL[Yellow]}${temp_warn}${GAUGESH}${idsCL[Default]} /${spcc}${idsCL[LightRed]}${temp_crit}${GAUGESH}=>${idsCL[Default]} ]"
	fi
}
AVERAGETEMP(){
	temptotal=0
	daysback=${2}
	sensorid=${1}
	logsfound=0;
	if [ "${3}" != "" ]; then
		entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${3}' AND sensorid='${sensorid}'"))
		entityid=${entityid[1]}
		WHERE="sensorid=${sensorid} AND entity=${entityid}"
	else
		WHERE="sensorid=${sensorid}"
	fi
		
	while IFS=$'\t' read date temp hum ;do
		temptotal=$(bc <<< "scale=1; ${temptotal}+${temp}")
		((logsfound++))
	done  < <(${mysql_conn} -se "USE servermonitor; SELECT date,temp,hum from sensor_data WHERE ${WHERE} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
	average=$(bc <<< "scale=1; ${temptotal}/${logsfound}")
	(( $(bc <<<"${average} < 1") )) && [[ "${average}" = *"."* ]] && average="0${average}"
	echo ${average}
	# echo ${logsfound}
}

SENDDAILY(){
	dailtemp=$(DAILYTEMP)
	SENDNOTICE "Daily Temp Readings" "${dailtemp}"
}

CHECKTEMP_SERVICE(){
	cw=20
	declare -A last_temp
	echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - 'Check Temp' Service Startup" >> ${PW_LOGFILE}
	while true; do
		# start=`date +%s`
		# time="$(date +%H)$(date +%M)"
		# if [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ]; then
		# 	if [ ! -f ${PW_TMPFOLDER}/.sentdaily ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.sentdaily)) -gt 600 ]; then
		# 		touch ${PW_TMPFOLDER}/.sentdaily
		# 		SENDDAILY &
		# 	fi
		# fi	
		for SENSORa in ${PW_SENSOR_ORD[@]}; do
			logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa}
			[ ! -f ${logtemp} ] && touch ${logtemp}
			[ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && relog=1 || relog=0
						
			if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]) || ([ "${PW_HOST_ADDRESSES[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_HOST_ADDRESSES[${SENSORa}]})" != "false" ]); then
			# if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then
				if [ -f ${PW_TMPFOLDER}/${SENSORa}.down ]; then
					SENDNOTICE "${SENSORa} Online" "${SENSORa} Sensor is back online"
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp}
					rm -f ${PW_TMPFOLDER}/${SENSORa}.down
				fi
				if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
					vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
					IFS=$'\n'
					read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
					unset IFS
					[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
					t=1
					
				else
					SENSORS_CHECK=(${SENSORa})
				fi
				if [ "${SENSORS_CHECK}" != "" ]; then
					for SENSOR in "${SENSORS_CHECK[@]}"; do
						if [ "${PW_THRESHOLDS[${SENSORa}]}" != "" ]; then
							echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 1 | read temp_warn
							echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 2 | read temp_crit	
							[ "$(echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3)" != "" ] && echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3 | read temp_crit_sys || temp_crit_sys=${temp_crit}
						fi
				
						if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
							mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
							echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
							echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
		
						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
							temp_c=$(vcgencmd measure_temp)
							temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
							temp_h=''
		
						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
							temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
							temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
							temp_h=''
				
						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
							temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
							temp_c=`echo "scale=1; ${temp_c}/10" | bc`
							temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
							temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
							temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc)
							temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
							temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
							temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc)
							temp_crit_sys=${temp_crit}
							temp_h=''
							vSENSOR=${SENSOR}
							SENSOR="${SENSORa}-${vSENSOR// /_}"
							((t++))
							
						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
							if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then
								temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
							else
								temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage")
							fi
							temp_h=''
							
						fi
						[ "${last_temp[${SENSOR//-/}]}" == "" ] && last_temp[${SENSOR//-/}]=0
						
						logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
						if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
							[ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ] && GAUGE="CPU-USAGE" || GAUGE="TEMP"
							[ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ] && GAUGESH='%' || GAUGESH="'F"
							if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
								SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
								echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
								rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
			
							fi
							
							[ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ] && temp_f=$(echo "scale=2; $temp_c/1" | bc -l) || temp_f=$(echo "scale=2; ${temp_c}*1.8 + 32" | bc)
							[ "${SENSOR}" == "ServerRoomTH" ] && temp_f=`echo "scale=2; ${temp_f}+${PW_ServerRoomTH_Dev}" | bc`
							# echo "HERE: [${SENSOR//-/}] - ${last_temp[${SENSOR//-/}]} - ${temp_c} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
							# [ ! "${temp_c}" ] || [ ! "${temp_f}" ] || [ ! "${temp_warn}" ] || [ ! "${temp_crit}" ] || [ ! "${temp_crit_sys}" ] && echo "VALUE NOT FOUND"
					
							if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
								echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
								echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
								echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
							fi
							
							if (( $(bc <<< "${temp_f} < ${temp_warn}") )); then
								if (( $(bc <<< "${last_temp[${SENSOR//-/}]} > 1") )); then
									SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH}
Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR//-/}]}${GAUGESH}" >> ${logtemp}
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE}
									# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
				
								elif [ "${last_temp[${SENSOR//-/}]}" == "0" ]; then
									# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
# NORMAL TEMP: ${temp_f}${GAUGESH}"
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp}
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
									# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
				
								elif [ "${relog}" == "1" ]; then
									echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp}
				
								fi
								rm -f ${PW_TMPFOLDER}/temp.*
								rm -f ${PW_TMPFOLDER}/power.${SENSORa}.off
								last_temp[${SENSOR//-/}]=1
								
								# iDS-vMS-Offsite Fan Issue Workaround
								if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
									vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
									# vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
									if [ "${vHOSTFAN}" != "" ]; then
										FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
										# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
										if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then
										# if [ ${FanSpeed} -ne 30 ]; then
											SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
										fi
									fi
								fi
								
							else
								([ ! "${last_temp[${SENSOR//-/}]}" ] || (( $(bc <<< "${last_temp[${SENSOR//-/}]} < 2") ))) && last_temp[${SENSOR//-/}]=${temp_f}
								temp_diff=`echo "scale=2; ${last_temp[${SENSOR//-/}]}-${temp_f}" | bc`
								[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`								
								
								if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
									touch ${PW_TMPFOLDER}/temp.warn
									if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
										SENDNOTICE "${SENSOR} ${GAUGE} WARNING" "WARNING ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
										last_temp[${SENSOR//-/}]=${temp_f}
										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - WARNING ${GAUGE} - (Difference of ${temp_diff}')" >> ${logtemp}
							
										# iDS-vMS-Offsite Fan Issue Workaround
										if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
											SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
											ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
										fi							
									fi
		
								elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then
									if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
										SENDNOTICE "${SENSOR} ${GAUGE} CRITICAL" "CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - CRITICAL - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp}
										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
										last_temp[${SENSOR//-/}]=${temp_f}
									
										if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f touch ${PW_TMPFOLDER}/temp.crit ]; then
											touch ${PW_TMPFOLDER}/temp.crit
											SHUTDOWN_MAIN ${SENSOR} &
											echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp}
										
										# elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
										# 	SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
										# 	SHUTDOWN_SERVER ${SENSORa} &
										# 	touch ${PW_TMPFOLDER}/power.${SENSORa}.off

										fi
									fi
								elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then
									if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
										SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
										echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
										last_temp[${SENSOR//-/}]=${temp_f}
										if [ ! -f touch ${PW_TMPFOLDER}/temp.critsys ]; then
											touch ${PW_TMPFOLDER}/temp.critsys
											[ "${SENSOR}" == "ServerRoomTH" ] && SHUTDOWN_SYS ${SENSOR} TEMP &
											SHUTDOWN_CRIT ${SENSOR} &
											echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
										fi
									fi
								fi
							fi
							
							[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
							if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
								wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}*60" | bc)
							else
								wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_NORMAL}*60" | bc)
							fi
							
							if [ "${PW_SENSOR_ID[${SENSORa}]}" != "" ] && [ ${lastinsert} -ge ${wait} ]; then
								if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
									entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
									if [ "${entityid[1]}" == "" ]; then
										${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
										entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
									fi
									entityid=${entityid[1]}
									
									QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
									if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ] && [ "${vHOSTFAN}" != "" ]; then
										${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')"
									fi
								elif [ "${temp_h}" != "" ]; then
									QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')"
								else
									QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
								fi
								${mysql_conn} -e "${QRY}"
								touch ${PW_TMPFOLDER}/${SENSOR}.insert
							fi
						else
							if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
								SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
								echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
								touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
							fi

						fi
				
					done
					
				fi
			
			elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down
				if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ]; then
					SENDNOTICE "${SENSORa} ERROR" "${SENSORa} Sensor is down" 1
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp}
					touch ${PW_TMPFOLDER}/${SENSORa}.down
			
				fi
					
			fi
			
		done
		
		# end=`date +%s`; runtime=$((end-start)); echo "runtime: ${runtime}, pausing for 60secs..."
		sleep 10		
	done # &
}

DAILYTEMP(){
	cw=35;
	echo
	echo -e "Sensor Temperature(s)              Now / 1DayAVG / 7DayAVG"
	DIVIDER . . 60
	for SENSORa in ${PW_SENSOR_ORD[@]}; do
		if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
			vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
			IFS=$'\n'
			read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
			unset IFS
			t=1
		else
			SENSORS_CHECK=(${SENSORa})
		fi
		
		for SENSOR in "${SENSORS_CHECK[@]}"; do
						
			if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
				mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
				temp_c=$(echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1)
		
			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
				temp_c=$(vcgencmd measure_temp)
				temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
			
			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
				temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
				temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
		
			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
				temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
				temp_c=`echo "scale=1; ${temp_c}/10" | bc`
				vSENSOR=${SENSOR}
				SENSOR="${SENSORa}-${vSENSOR// /_}"
				((t++))
			else
				temp_c=''
		
			fi
			if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
				average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1 ${vSENSOR// /_})
				average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7 ${vSENSOR// /_})
			else
				average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1)
				average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7)
			fi
						
			c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
			if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
				temp_f=`echo "scale=2; ${temp_c}*1.8 + 32" | bc`
				echo -e "${SENSOR}${spc}${temp_f}'F / ${average1}'F / ${average7}'F"
				
			else
				echo -e "${SENSOR}${spc}No Data"
			
			fi
		done
		[ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ] && echo
	done
}

GET_LACROSSE_INFO(){
	
	INFO=$(/bin/python3 ${PW_INCFOLDER}/getInfo.lacrosse.py "${PW_LACROSSE_ID[${1}]}" | sed -e "s/\[{'//g" | sed -e "s/}\]//g" | sed -e "s/', '/~/g" | sed -e "s/': '/:/g" | sed -e "s/': /:/g")
	IFS=$'~'; read -rd '' -a INFO <<<"${INFO}"; unset IFS
	declare -A DEVICE_INFO
	for DINFO in "${INFO[@]}"; do
		key=${DINFO%%:*}
		val=${DINFO##*:}; val=${val//[$'\t\r\n ']}
		DEVICE_INFO[${key}]=${val}
		# echo "'${key}' => '${val}'"
	done
	
	echo ${DEVICE_INFO['probe_temp']}
	
}