#!/usr/bin/env bash
# powerwall - CLI commands to control VM guest power

action="${1}"

source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/powerwall/defaults.inc
source ${PW_FOLDER}/settings.conf

# TEST=true

shopt -s lastpipe

VERBOSE=false


CHECKTEMP(){
	start=`date +%s`
	cw=25;
	echo
	
	[ "${1}" != "" ] && avgdays=${1} || avgdays=noavg
	if [ "${avgdays}" != "noavg" ]; then
		[ ${avgdays} -lt 10 ] && avgdays_disp=" ${avgdays}" || avgdays_disp=${avgdays}
		echo -e "${idsCL[Cyan]}Sensor Temperature(s)               Current / ${avgdays_disp}DayAVG   ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[Red]}critical${idsCL[Default]}"
	else
		echo -e "${idsCL[Cyan]}Sensor Temperature(s)               Current    ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[Red]}critical${idsCL[Default]}"
	fi
	DIVIDER . yellow 90
		
	for SENSOR in ${PW_SENSOR_ORD[@]}; do
		if [ "${1}" == "" ] || [[ "${SENSOR}" = *"${1}"* ]] || [ "${1}" == "noavg" ] || [[ "${1}" =~ ^[0-9]+$ ]]; then
			
			if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "esxi" ]; then
				vHOSTiDRACIP=${PW_HOST_IDRACS[${SENSOR}]}
				vHOSTIP=${PW_HOST_ADDRESSES[${SENSOR}]}				
				vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTiDRACIP}]}
				echo -e "\r\033[K${idsCL[LightCyan]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[LightCyan]}[${vHOSTiDRACIP}]${idsCL[Default]}"
				echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... "
				
				if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then
					vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
					
					# iDS-vMS-Offsite Fan Issue Workaround
					if [ "${vHOSTiDRACIP}" == "10.2.1.21" ]; then
						# vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.1 -O vq)
						# FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
						vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTiDRACIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
						FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
						if [ ${FanSpeed} -gt 92 ]; then
							SENDNOTICE "${vHOSTiDRACIP} Fan Issue" "Workaround commands sent to set the Fans at 30%" 1
							ipmitool -I lanplus -H ${vHOSTiDRACIP} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
							ipmitool -I lanplus -H ${vHOSTiDRACIP} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
						fi

					fi
					
					IFS=$'\n'
					read -rd '' -a vHOST_TEMPNAMES <<<"${vHOST_TEMPNAMES}"
					unset IFS
					echo -en "\r\033[K"
					if [ ${#vHOST_TEMPNAMES[@]} > 1 ]; then
						t=1
						for sensorname in "${vHOST_TEMPNAMES[@]}"; do
							temp_c=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
							temp_c=`echo "scale=1; ${temp_c}/10" | bc`
							temp_warn=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
							temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
							temp_crit=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
							temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
							
							c=0; spc=''; spc1=`expr ${cw} - ${#sensorname} - 1`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
							
							[ "${avgdays}" != "noavg" ] && echo -en "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${idsCL[LightYellow]}Calculating ${avgdays} Day average ... "
							reading=$(CHECKTEMPSENSOR ${SENSOR} ${temp_c} ${temp_warn} ${temp_crit} ${sensorname// /_} ${avgdays})
							[ "${avgdays}" != "noavg" ] && echo -en "\r\033[K"
							echo -e "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${reading}"
							
							((t++))
						done
					else
						echo -e "${idsCL[LightCyan]} ${vHOSTNAME}${spc}${idsCL[Default]}: No Data"
					fi
					
				else #host down
					echo -e "\r\033[K${idsCL[Red]}${SENSOR} Host is down"
					
					# if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
					# 	SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1
					# 	echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
					# 	touch ${PW_TMPFOLDER}/${EHOST}.down
					# fi
				fi
				
			else
				[ "${avgdays}" != "noavg" ] && echo -en "${idsCL[Yellow]}Pulling data and calculating averages for '${SENSOR}' ... " || echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... "
				reading=$(CHECKTEMPSENSOR ${SENSOR} . . . . ${avgdays})
				c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
				echo -e "\r\033[K${idsCL[LightCyan]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${reading}"
			fi
			echo
		fi
	done
	end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
	[ "${action}" != "" ] && echo
}

CHECKTEMPSENSOR(){
	echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
	echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
	if [ "${6}" != "noavg" ]; then
		[ "${6}" != "" ] && [[ "${6}" =~ ^[0-9]+$ ]] && avgdays=${6} || avgdays=3
		if [ "${PW_SENSOR_TYPE[${1}]}" != "esxi" ]; then
			average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays})
		else
			average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays} ${5})
		fi
	else
		average=''
	fi
	
	if [ "${PW_SENSOR_TYPE[${1}]}" == "mqtt" ]; then
		mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
		echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
		
	elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ]; then
		temp_c=$(vcgencmd measure_temp)
		temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
		
	elif [ "${PW_SENSOR_TYPE[${1}]}" == "remotesystem" ]; then
		temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
		if [ "${temp_c}" != "" ]; then
			temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
			
		fi
	elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then
		temp_c=${2}
		temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'`
		temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'`

	fi
	
	if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
		temp_f=$(bc <<< "scale=1; ${temp_c}*1.8+32")
		#echo "${temp_c} -> ${temp_f}"
		c=0; spct=''; spc1=`expr 6 - ${#temp_f}`; until [ ${c} = ${spc1} ]; do spct="${spct} "; c=`expr ${c} + 1`; done
		
		if [ "${average}" != "" ]; then
			c=0; spca=''; spc1=`expr 6 - ${#average}`; until [ ${c} = ${spc1} ]; do spca="${spca} "; c=`expr ${c} + 1`; done
			if [ "${average}" == "" ]; then
				averagedisp="${idsCL[Green]}"
			elif [ $(bc -l <<< "${average} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${average} < ${temp_crit}") -eq 1 ]; then
				averagedisp="${idsCL[Yellow]}${average}'F"
			elif [ $(bc -l <<< "${average} >= ${temp_crit}") -eq 1 ]; then
				averagedisp="${idsCL[Red]}${average}'F"
			else
				averagedisp="${idsCL[Green]}${average}'F"
			fi
		fi
		
		if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
			[ "${average}" != "" ] && echo -en "${idsCL[Yellow]}WARNING ${spct}${temp_f}'F /${spca}${averagedisp}${idsCL[Default]}" || echo -en "${idsCL[Yellow]}WARNING ${spct}${temp_f}'F${idsCL[Default]}"
		elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ]; then
			[ "${average}" != "" ] && echo -en "${idsCL[Red]}CRITICAL${spct}${temp_f}'F /${spca}${averagedisp}${idsCL[Default]}" || echo -en "${idsCL[Red]}CRITICAL${spct}${temp_f}'F${idsCL[Default]}"
		else
			[ "${average}" != "" ] && echo -en "${idsCL[Green]}Normal  ${spct}${temp_f}'F /${spca}${averagedisp}${idsCL[Default]}" || echo -en "${idsCL[Green]}Normal  ${spct}${temp_f}'F${idsCL[Default]}"
		fi
		tmp="${spct}${temp_f}"
		c=0; spc=''; spc1=`expr 10 - ${#tmp}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
		[ ${#temp_warn} == 2 ] && spcw='  ' || spcw=' '
		[ ${#temp_crit} == 2 ] && spcc='  ' || spcc=' '
		echo -e "${spc}[${spcw}${idsCL[Yellow]}${temp_warn}'F${idsCL[Default]} /${spcc}${idsCL[Red]}${temp_crit}'F=>${idsCL[Default]} ]"
	fi
}
AVERAGETEMP(){
	temptotal=0
	daysback=${2}
	sensorid=${1}
	logsfound=0;
	if [ "${3}" != "" ]; then
		entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${3}' AND sensorid='${sensorid}'"))
		entityid=${entityid[1]}
		WHERE="sensorid=${sensorid} AND entity=${entityid}"
	else
		WHERE="sensorid=${sensorid}"
	fi
		
	while IFS=$'\t' read date temp hum ;do
		temptotal=$(bc <<< "scale=1; ${temptotal}+${temp}")
		((logsfound++))
	done  < <(${mysql_conn} -se "USE servermonitor; SELECT date,temp,hum from sensor_data WHERE ${WHERE} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
	average=$(bc <<< "scale=1; ${temptotal}/${logsfound}")
	echo ${average}
}

SENDDAILY(){
	dailtemp=$(DAILYTEMP)
	SENDNOTICE "Daily Temp Readings" "${dailtemp}"
}

CHECKTEMP_SERVICE(){
	cw=20
	declare -A last_temp
	while true; do
		# start=`date +%s`
		time="$(date +%H)$(date +%M)"
		if [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ]; then
			if [ ! -f ${PW_TMPFOLDER}/.sentdaily ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.sentdaily)) -gt 600 ]; then
				touch ${PW_TMPFOLDER}/.sentdaily
				SENDDAILY &
			fi
		fi		
		for SENSORa in ${PW_SENSOR_ORD[@]}; do
			logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa}
			[ ! -f ${logtemp} ] && touch ${logtemp}
			[ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge 3600 ] && relog=1 || relog=0
						
			if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]) || ([ "${PW_HOST_ADDRESSES[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_HOST_ADDRESSES[${SENSORa}]})" != "false" ]); then
			# if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then
				if [ -f ${PW_TMPFOLDER}/${SENSORa}.down ]; then
					SENDNOTICE "${SENSORa} Online" "${SENSORa} Sensor is back online"
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp}
					rm -f ${PW_TMPFOLDER}/${SENSORa}.down
				fi
				if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
					vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
					IFS=$'\n'
					read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
					unset IFS
					[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
					t=1
					
					# iDS-vMS-Offsite Fan Issue Workaround
					if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
						# vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.1 -O vq)
						# FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
						vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
						FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
						if [ ${FanSpeed} -gt 92 ]; then
							SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Workaround commands sent to set the Fans at 30%25" 1
							ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
							ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
							FanSpeed=30
						fi
					fi
					
				else
					SENSORS_CHECK=(${SENSORa})
				fi
				if [ "${SENSORS_CHECK}" != "" ]; then
					for SENSOR in "${SENSORS_CHECK[@]}"; do
				
						if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
							mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
							echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
							echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
		
						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
							temp_c=$(vcgencmd measure_temp)
							temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
							temp_h=''
		
						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
							temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
							temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
							temp_h=''
				
						elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
							temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
							temp_c=`echo "scale=1; ${temp_c}/10" | bc`
							temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
							temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
							temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc)
							temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
							temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
							temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc)
							temp_crit_sys=${temp_crit}
							temp_h=''
							vSENSOR=${SENSOR}
							SENSOR="${SENSORa}-${vSENSOR// /_}"
							((t++))
						fi
						
						logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
						if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
							if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
								SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
								echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
								rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
			
							fi					
							temp_f=$(echo "scale=2; ${temp_c}*1.8 + 32" | bc)
							# echo "HERE: ${SENSOR} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
					
							if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
								echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
								echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
								echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
							fi
					
							if [ "${last_temp[${SENSOR}]}" != "" ]; then
								temp_diff=`echo "scale=2; ${last_temp[${SENSOR}]}-${temp_f}" | bc`
								[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`
								
							else
								last_temp[${SENSOR}]=0
								temp_diff=0
							fi
			
							if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
								touch ${PW_TMPFOLDER}/temp.warn
								if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
									SENDNOTICE "${SENSOR} TEMP WARNING" "WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
									last_temp[${SENSOR}]=${temp_f}
									echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
							
									# iDS-vMS-Offsite Fan Issue Workaround
									if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
										SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
									fi							
								fi
		
							elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then
								touch ${PW_TMPFOLDER}/temp.crit
								if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
									SENDNOTICE "${SENSOR} TEMP CRITICAL" "CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
									echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
									last_temp[${SENSOR}]=${temp_f}
									
									if [[ "${PW_OFFSITEHOSTS}" != *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then
										SHUTDOWN_MAIN ${SENSOR} &
										echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}

									elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
										SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
										SHUTDOWN_SERVER ${SENSORa} &
										touch ${PW_TMPFOLDER}/power.${SENSORa}.off

									fi	
								fi
							elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then
								touch ${PW_TMPFOLDER}/temp.critsys
								if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
									SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
									echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
									last_temp[${SENSOR}]=${temp_f}
									SHUTDOWN_SYS ${SENSOR} TEMP &
									SHUTDOWN_CRIT ${SENSOR} &
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
								fi
							else
			
								if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
									SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE}
									[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
				
								elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
									# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
		# NORMAL TEMP: ${temp_f}'F"
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
									echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
									[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
				
								elif [ ${relog} -eq 1 ]; then
									echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Normal Temp" >> ${logtemp}
				
								fi
								rm -f ${PW_TMPFOLDER}/temp.*
								rm -f ${PW_TMPFOLDER}/power.${SENSORa}.off
								last_temp[${SENSOR}]=1
								
								# iDS-vMS-Offsite Fan Issue Workaround
								if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
									# vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.1 -O vq)
									# FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
									vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
									FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
									if [ ${FanSpeed} -ne 30 ]; then
										SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
										ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
									fi
								fi
							fi
		
							[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
							if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
								wait=60
							else
								wait=300
							fi
							if [ ${lastinsert} -ge ${wait} ]; then
								if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
									entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
									if [ "${entityid[1]}" == "" ]; then
										${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
										entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
									fi
									entityid=${entityid[1]}
									
									QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
									if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ]; then
										QRY2="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')"
										${mysql_conn} -e "${QRY2}"
									fi
								elif [ "${temp_h}" != "" ]; then
									QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')"
								else
									QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
								fi
								${mysql_conn} -e "${QRY}"
								touch ${PW_TMPFOLDER}/${SENSOR}.insert
							fi
						else
							if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
								SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
								echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
								touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
							fi

						fi
				
					done
					
					
					
				fi
			
			elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down
				if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ]; then
					SENDNOTICE "${SENSORa} ERROR" "${SENSORa} Sensor is down" 1
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp}
					touch ${PW_TMPFOLDER}/${SENSORa}.down
			
				fi
					
			fi
			
		done
		
		# end=`date +%s`; runtime=$((end-start)); echo "runtime: ${runtime}, pausing for 60secs..."
		sleep 60		
	done # &
}



DAILYTEMP(){
	cw=35;
	echo
	echo -e "Sensor Temperature(s)              Now / 1DayAVG / 7DayAVG"
	DIVIDER . . 60
	for SENSORa in ${PW_SENSOR_ORD[@]}; do
		if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
			vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
			IFS=$'\n'
			read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
			unset IFS
			t=1
		else
			SENSORS_CHECK=(${SENSORa})
		fi
		
		for SENSOR in "${SENSORS_CHECK[@]}"; do
						
			if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
				mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
				temp_c=$(echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1)
		
			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
				temp_c=$(vcgencmd measure_temp)
				temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
			
			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
				temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
				temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
		
			elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
				temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
				temp_c=`echo "scale=1; ${temp_c}/10" | bc`
				vSENSOR=${SENSOR}
				SENSOR="${SENSORa}-${vSENSOR// /_}"
				((t++))
			else
				temp_c=''
		
			fi
			if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
				average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1 ${vSENSOR// /_})
				average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7 ${vSENSOR// /_})
			else
				average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1)
				average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7)
			fi
						
			c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
			if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
				temp_f=`echo "scale=2; ${temp_c}*1.8 + 32" | bc`
				echo -e "${SENSOR}${spc}${temp_f}'F / ${average1}'F / ${average7}'F"
				
			else
				echo -e "${SENSOR}${spc}No Data"
			
			fi
		done
		[ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ] && echo
	done
}


CHECKPOWER(){
	start=`date +%s`
	rm -f ${PW_TMPFOLDER}/*.hostpower*

	if [ "${1}" == "" ] || [ "${1}" == "mainsite" ]; then
		echo
		DIVIDER . yellow 75
		echo -e "${idsCL[Yellow]}Mainsite Power Information"
		DIVIDER . yellow 75
		
		INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
		if [ "${INVSTATSINFO}" != "" ]; then
			INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
			INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
			OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
			OUTPUTV2=`echo "scale=2; ${OUTPUTV}/2" | bc`
			OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
			OUTPUTA2=`echo "scale=2; ${OUTPUTA}*2" | bc`
			OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
			[[ "${OUTPUTW}" == *"."* ]] && [[ "${OUTPUTW}" != *".0"* ]] && OUTPUTW=`printf "%'.2f\n" ${OUTPUTW}` || OUTPUTW=`printf "%'.0f\n" ${OUTPUTW}`
			BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
			BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
			[[ "${BATTA}" == *"-"* ]] && BATTA_disp="${idsCL[Green]}${BATTA}'amps - Charging" || BATTA_disp="${idsCL[LightRed]}${BATTA}'amps - Discharging"
		fi

		echo -e "${idsCL[LightCyan]}Inverter Input Volts     ${idsCL[Default]}: ${idsCL[Green]}${INPUTV}'volts"
		echo -e "${idsCL[LightCyan]}Inverter Input Amps      ${idsCL[Default]}: ${idsCL[Green]}${INPUTA}'amps"
		echo
		echo -e "${idsCL[LightCyan]}Inverter Output Volts    ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTV}'volts / ${OUTPUTV2}'volts"
		echo -e "${idsCL[LightCyan]}Inverter Output Amps     ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTA}'amps / ${OUTPUTA2}'amps"
		echo -e "${idsCL[LightCyan]}Inverter Output Watts    ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTW}'watts"
		echo
		echo -e "${idsCL[LightCyan]}Battery Voltage          ${idsCL[Default]}: ${idsCL[Green]}${BATTV}'volts${idsCL[Default]}"
		echo -e "${idsCL[LightCyan]}Battery Amps Chg/DChg    ${idsCL[Default]}: ${BATTA_disp}${idsCL[Default]}"
		echo


		# if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
			SVRRM_VOLT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
			SVRRM_VOLT=`echo "scale=2; ${SVRRM_VOLT}/10" | bc`
			SVRRM_WATT=$(snmpwalk  -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
			[[ "${SVRRM_WATT}" == *"."* ]] && [[ "${SVRRM_WATT}" != *".0"* ]] && SVRRM_WATT_disp=`printf "%'.2f\n" ${SVRRM_WATT}` || SVRRM_WATT_disp=`printf "%'.0f\n" ${SVRRM_WATT}`
			SVRRM_AMP=$(snmpwalk  -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
			SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
		# fi
		#
		# if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
			APCPDU_AMP=$(snmpwalk  -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
			APCPDU_AMP=`echo "scale=2; ${APCPDU_AMP}/10" | bc`
			# APCPDU_WATT=$(snmpwalk  -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.16.0 | sed 's/.*: //')
			APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
			[[ "${APCPDU_WATT}" == *"."* ]] && [[ "${APCPDU_WATT}" != *".0"* ]] && APCPDU_WATT_disp=`printf "%'.2f\n" ${APCPDU_WATT}` || APCPDU_WATT_disp=`printf "%'.0f\n" ${APCPDU_WATT}`
		# fi

		echo -e "${idsCL[LightCyan]}ServerRack Amps          ${idsCL[Default]}: ${idsCL[Green]}${APCPDU_AMP}'amps${idsCL[Default]}"
		echo -e "${idsCL[LightCyan]}ServerRack Wattage       ${idsCL[Default]}: ${idsCL[Green]}${APCPDU_WATT_disp}'watts ~ estimate${idsCL[Default]}"
		echo

		if [ "${SVRRM_VOLT}" != "" ]; then
			if [ "${APCPDU_AMP}" != "" ]; then
				NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
				NETRK_WATT=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
				[[ "${NETRK_WATT}" == *"."* ]] && [[ "${NETRK_WATT}" != *".0"* ]] && NETRK_WATT=`printf "%'.2f\n" ${NETRK_WATT}` || NETRK_WATT=`printf "%'.0f\n" ${NETRK_WATT}`
			fi
			VOLT_DROP=`echo "scale=1; ${OUTPUTV2}-${SVRRM_VOLT}" | bc`
		fi

		echo -e "${idsCL[LightCyan]}NetworkRack Amps         ${idsCL[Default]}: ${idsCL[Green]}${NETRK_AMP}'amps ~ estimate${idsCL[Default]}"
		echo -e "${idsCL[LightCyan]}NetworkRack Wattage      ${idsCL[Default]}: ${idsCL[Green]}${NETRK_WATT}'watts ~ estimate${idsCL[Default]}"
		echo

		echo -e "${idsCL[LightCyan]}Voltage                  ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLT}'volts${idsCL[Default]}"
		echo -e "${idsCL[LightCyan]}Voltage Drop             ${idsCL[Default]}: ${idsCL[LightRed]}${VOLT_DROP}'volts${idsCL[Default]}"
		echo -e "${idsCL[LightCyan]}Total Amps               ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_AMP}'amps${idsCL[Default]}"
		echo -e "${idsCL[LightCyan]}Total Wattage            ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_WATT_disp}'watts${idsCL[Default]}"
		echo
		
	fi
	
	
	if [ "${1}" == "" ] || [ "${1}" == "offsite" ]; then
		
			
		DIVIDER . yellow 75
		echo -e "${idsCL[Yellow]}Offsite Power Information"
		DIVIDER . yellow 75
	
		upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
		if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
			OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
			OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
			OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
			OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
			OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
			OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
			OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
			OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
			OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
		
			# OFFSITE_VOLTIN=$(snmpwalk  -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.3.2.1.0 | sed 's/.*: //')
			# OFFSITE_VOLTIN=`echo "scale=2; ${OFFSITE_VOLTIN}/10" | bc`
			# OFFSITE_VOLTOUT=$(snmpwalk  -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.1.0 | sed 's/.*: //')
			# OFFSITE_VOLTOUT=`echo "scale=2; ${OFFSITE_VOLTOUT}/10" | bc`
			# OFFSITE_LOAD=$(snmpwalk  -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.3.0 | sed 's/.*: //')
			# OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
			# OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*900" | bc`
			# OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
			# OFFSITE_BATT_RUNTIME=$(snmpwalk  -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.2.1.33.1.2.3.0 | sed 's/.*: //')
			# OFFSITE_BATT_CAP=$(snmpwalk  -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.2.1.33.1.2.4.0 | sed 's/.*: //')
		
		fi
	
		echo -e "${idsCL[LightCyan]}UPS Voltage In           ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_VOLTIN}'volts${idsCL[Default]}"
		echo -e "${idsCL[LightCyan]}UPS Voltage Out          ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_VOLTOUT}'volts${idsCL[Default]}"
		echo
		echo -e "${idsCL[LightCyan]}Total Amps               ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_AMPS}'amps ~ estimate${idsCL[Default]}"
		echo -e "${idsCL[LightCyan]}Total Wattage            ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_WATTS}'watts ~ estimate${idsCL[Default]}"
		echo
		echo -e "${idsCL[LightCyan]}Battery Capacity         ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_BATT_CAP}%${idsCL[Default]}"
		echo -e "${idsCL[LightCyan]}Battery Runtime          ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_BATT_RUNTIME%.*}'Mins${idsCL[Default]}"
		echo				
	fi
	
				
	# if [ "${1}" == "" ] || [ "${1}" == "hosts" ]; then
		DIVIDER . yellow 75
		echo -e "${idsCL[Yellow]}ESXi Host Power Information"
		DIVIDER . yellow 75
		
		for SENSOR in ${PW_SENSOR_ORD[@]}; do
			if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
				gochk=0
				if [ "${1}" == "" ] || [ "${1}" == "hosts" ] || ([ "${1}" == "mainsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.10."* ]]) || ([ "${1}" == "offsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.2.1."* ]]); then
					gochk=1
				fi
				if [ "${1}" == "offsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.2."* ]]; then
					echo yes
				else
					echo no
				fi
				if [ ${gochk} -eq 1 ]; then
					HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.3 | sed 's/.*: //')
					if [ "${HOST_WATTS}" = "" ]; then
						HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq | sed -e 's/"//g')
						IFS=$'\n'
						read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}"
						unset IFS
						s=1
						for ss in "${HOST_WATT_SENSORS[@]}"; do
							[[ "${ss}" = *"System Board"* ]] && break || ((s++))
						done				
						HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
					fi
					[[ "${HOST_WATTS}" = *"Timeout"* ]] && HOST_WATTS_disp="${idsCL[LightRed]}Host Offline" || HOST_WATTS_disp="${idsCL[Green]}${HOST_WATTS}'watts"
					echo -e "${idsCL[LightCyan]}${SENSOR}            ${idsCL[Default]}: ${HOST_WATTS_disp}${idsCL[Default]}"
				fi
			fi
		done
		
	# fi
	
	end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
	[ "${action}" != "" ] && echo
}
		
CHECKPOWER_SERVICE(){
	voltstatus=0
	battstatus=0
	while true; do
		INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
		SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
		
		if (( $(bc <<<"${SVRRM_UPS_INPUTACV} < ${min_acvolt}") )); then
			if [ ! -f ${PW_TMPFOLDER}/power.ups.low ]; then
				touch ${PW_TMPFOLDER}/power.ups.low
				
			elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then
				SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1
				
			fi
			
		elif [ -f ${PW_TMPFOLDER}/power.ups.low ]; then
			rm -f ${PW_TMPFOLDER}/power.ups.low
		fi
		
		if [ "${INVSTATSINFO}" != "" ]; then
			INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
			BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV')
			
			if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then
				if [ ${voltstatus} -eq 3 ]; then
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Back to Normal Voltage" >> ${PW_LOGPOWER}
					echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
					SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected
VOLTAGE: ${INPUTACV}'volts"
					battstatus=0
				else
					[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts - Normal Voltage" >> ${PW_LOGPOWER}
					if [ ${voltstatus} -eq 0 ]; then
						SENDNOTICE "Power Nominal" "Service Startup
Normal voltage detected
VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: ${BATTVOLT}'volts"
						echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
						echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Service Startup - Normal Voltage" >> ${PW_LOGFILE}
					fi
				fi
				voltstatus=1
				rm -f ${PW_TMPFOLDER}/power.ac.*
			else
				echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
				echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - POWER OFF/LOW POWER" >> ${PW_LOGFILE}
				if [ ${voltstatus} -lt 3 ]; then
					echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power off or low voltage detected" >> ${PW_LOGFILE}
					echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power off or low voltage detected" >> ${PW_LOGPOWER}
					SENDNOTICE "POWER ALERT - POWER OFF/LOW POWER!!" "Power off or low voltage detected
VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: ${BATTVOLT}'volts" 1
					last_battvolt=${BATTVOLT}
					voltstatus=3
					touch ${PW_TMPFOLDER}/power.ac.low
				fi
				if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge 180 ]; then
					echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGFILE}
					SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1
					SHUTDOWN_SERVER 10.10.2.10 &
					touch ${PW_TMPFOLDER}/power.ac.conserve
				fi
			fi
			
			if (( $(bc <<<"${BATTVOLT} < ${min_battvolt} && ${BATTVOLT} > ${min_battvolt_sys}") )); then
				echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
				echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
				touch ${PW_TMPFOLDER}/power.dc.low
				if (( $(bc <<<"${battstatus} < 2 && ${INPUTACV} < ${min_acvolt}") )); then
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
					SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
Shutting down main servers" 1
					SHUTDOWN_MAIN POWER &
					battstatus=2
				fi
			
			elif (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then
				echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
				echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
				touch ${PW_TMPFOLDER}/power.dc.low
				if (( $(bc <<<"${battstatus} < 3 && ${INPUTACV} < ${min_acvolt}") )); then
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
					SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts
Shutting down all servers" 1
					SHUTDOWN_SYS POWER &
				
					battstatus=3
					
				fi
			
			else
				[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage" >> ${PW_LOGPOWER}
				if [ ${battstatus} -eq 0 ]; then
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
				fi
				battstatus=1
			
			fi
			
			volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc`
			(( $(bc <<<"${volt_diff} < 0") )) && temp_diff=`echo "scale=1; ${volt_diff} * -1" | bc`
			if (( $(bc <<<"${volt_diff} > 0.5") )); then
				SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts"
				echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER}
				last_battvolt=${BATTVOLT}
			fi
		else
			sleep 20s
		fi
		
		######offsite check
		
# 		if [ "$(CHECK_HOST 10.2.1.1)" != "false" ]; then
# 			upsinfo=$(ssh admin@10.2.1.1 'upsc CP1500PFCLCD')
# 			OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
# 			OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
# 			OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
# 			OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
#
# 			if (( $(bc <<<"${OFFSITE_VOLTIN} >= ${min_acvolt}") )); then
# 				if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
# 					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - Offsite Back to Normal Voltage" >> ${PW_LOGPOWER}
# 					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
# 					SENDNOTICE "OFFSITE POWER NOMINAL AGAIN" "Normal voltage detected
# VOLTAGE: ${OFFSITE_VOLTIN}'volts"
# 					rm -f ${PW_TMPFOLDER}/power.offsite.ac.*
# 				fi
# 				if [ -f ${PW_TMPFOLDER}/power.offsite.off ]; then
# 					if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then
# 						touch ${PW_TMPFOLDER}/power.offsite.restored
#
# 					elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.restored)) -ge 60 ]; then
# 						SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back on after power was restored"
# 						for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
# 							HOSTPOWER ${offsite_host} on >/dev/null 2>&1
# 						done
# 						rm -f ${PW_TMPFOLDER}/power.offsite.*
# 					fi
#
# 				else
# 					[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -le 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - Offsite Normal Voltage" >> ${PW_LOGPOWER}
# 				fi
# 			else
# 				echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
# 				echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGFILE}
#
# 				if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge 240 ]; then
# 					SENDNOTICE "OFFSITE POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected
# BATT CAPACITY: ${OFFSITE_BATT_CAP}'%25
# BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'Mins" 1
# 					touch ${PW_TMPFOLDER}/power.offsite.ac.low
# 				fi
#
# 				if (( $(bc <<<"${OFFSITE_BATT_CAP} <= 95") )) && [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
# 					SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1
# 					SHUTDOWN_OFFSITE &
# 					touch ${PW_TMPFOLDER}/power.offsite.off
# 				fi
# 			fi
#
#
# 		fi
		
	    sleep 10s
	done # & 
}

POWERLOGGER_SERVICE(){
	while true; do
		LogDataTime=`date +'%Y-%m-%d %H:%M:%S'`
		
		INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
		if [ "${INVSTATSINFO}" != "" ]; then
			INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
			INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
			OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
			OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
			OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
			BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
			BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
			
			QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${LogDataTime}','${INPUTV}','${INPUTA}')"
			${mysql_conn} -e "USE servermonitor; ${QRY}"
			QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('4','${LogDataTime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')"
			${mysql_conn} -e "USE servermonitor; ${QRY}"
			QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${LogDataTime}','${BATTV}','${BATTA}')"
			${mysql_conn} -e "USE servermonitor; ${QRY}"
			
			QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${LogDataTime}','${INVSTATSINFO}')"
			${mysql_conn} -e "USE servermonitor; ${QRY}"
		fi
		
		if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
			SVRRM_VOLT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
			SVRRM_VOLT=`echo "scale=1; ${SVRRM_VOLT}/10" | bc`
			SVRRM_WATT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
			SVRRM_AMP=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
			SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
			
			QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('11','${LogDataTime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')"
			${mysql_conn} -e "USE servermonitor; ${QRY}"
		fi

		if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
			APCPDU_AMP=$(snmpwalk -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
			APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc`
			if [ "${SVRRM_VOLT}" != "" ]; then
				APCPDU_WATT=`echo "scale=1; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
				QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('12','${LogDataTime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')"
			else
				QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${LogDataTime}','${APCPDU_AMP}')"
			fi
			${mysql_conn} -e "USE servermonitor; ${QRY}"
		fi
		
		if [ "${SVRRM_VOLT}" != "" ] && [ "${APCPDU_AMP}" != "" ]; then
			NETRK_AMP=`echo "scale=1; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
			NETRK_WATT=`echo "scale=0; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
			
			QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('13','${LogDataTime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')"
			${mysql_conn} -e "USE servermonitor; ${QRY}"
		fi
		
		upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
		if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
			OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
			OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
			OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
			OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
			OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
			
			QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('15','${LogDataTime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')"
			${mysql_conn} -e "USE servermonitor; ${QRY}"
		fi
		
		if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*")" != "" ]; then
			sleep 1m
		else
			sleep 5m
		fi

	done
}




RESTARTALL_SERVICES(){
	STOPALL_SERVICES
	sleep 10s
	STARTALL_SERVICES
}
STOPALL_SERVICES(){
	for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
		SERVICE_MNG ${srvc} stop
	done
}
STARTALL_SERVICES(){
	for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
		SERVICE_MNG ${srvc} start
	done
}
SERVICE_MNG(){
	if [ "${2}" = "start" ]; then
		sa=Starting
		
	elif [ "${2}" = "stop" ]; then
		sa=Stopping

	elif [ "${2}" = "restart" ]; then
		sa=Restarting

	elif [ "${2}" = "status" ]; then
		sa='Status for'

	fi
	echo -en "${idsCL[LightCyan]}${sa} ${PW_POWERWALL_SERVICES[${1}]} Service ... "
	/bin/systemctl ${2} ${PW_POWERWALL_SERVICES[${1}]}
	echo -e "${idsCL[Green]}Done${idsCL[Default]}"
}
STOP_SERVICE(){
	echo "stop: ${1}"
	ps -ef | grep "powerwall.*service ${1} start" | grep -v "grep" | awk '{print $2}' | xargs kill
}

CHECK_SERVICES(){
	echo
	echo -e "${idsCL[Yellow]}PowerWall Monitor Service Status"
	DIVIDER . yellow 75
	
	for srvc in "${PW_POWERWALL_SERVICES[@]}"; do
		c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
		echo -en "${idsCL[LightCyan]}${srvc}${spc}: "
		[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
		echo -e "${idsCL[Default]}"
	done
	
	echo
}

SHUTDOWN_MAIN(){
	if [ "${1}" == "ServerRoomTH"] || [ "${1}" == "TEMP"]; then
		SHUTDOWN_SERVERS MAIN ${1}
		reason='Server Room Overheated!'
		sendnotice=true
		
	elif [ "${1}" == "POWER"]; then
		SHUTDOWN_SERVERS MAIN ${1}
		reason='Low Power!'
		sendnotice=true
		
	else
		sendnotice=false
	fi
	
	if [ "${sendnotice}" == "true"]; then
		SENDNOTICE "MAIN SERVERS SHUTDOWN" "MAIN servers have been shutdown
${reason}" 1
		echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
	fi
	
	echo
}
SHUTDOWN_SYS(){
	if [ "${1}" == "ServerRoomTH"] || [ "${1}" == "TEMP"]; then
		SHUTDOWN_SERVERS SYS ${1}
		reason='Server Room Overheated!'
		sendnotice=true
		
	elif [ "${1}" == "POWER"]; then
		SHUTDOWN_SERVERS SYS ${1}
		reason='Low Power!'
		sendnotice=true
		
	else
		sendnotice=false
	fi
	
	if [ "${sendnotice}" == "true"]; then
		SENDNOTICE "SYS SERVERS SHUTDOWN" "SYS servers have been shutdown
		${reason}" 1
		echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
		echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
	fi
	
	# curl --data "cmd=PWRoff" http://${PW_INVERTER_IP}/cmd
	echo
}
SHUTDOWN_CRIT(){
	if [ "${1}" == "Powerwall-CPU" ]; then
		SENDNOTICE "${SENSOR} TEMP CRITICAL" "Powerwall System shutting down" 1
		echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
		echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE}
		sleep 1s
		/sbin/poweroff
		
	elif [ "${1}" == "OctoPI-CPU" ]; then
		SENDNOTICE "${SENSOR} TEMP CRITICAL" "OctoPI System shutting down" 1
		echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - OctoPI System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
		echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} TEMP CRITICAL - OctoPI System shutting down" >> ${PW_LOGFILE}
		sleep 1s
		ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff
		
	fi
}
SHUTDOWN_OFFSITE(){
	if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
		SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down offsite host(s)" 1
		for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
			SHUTDOWN_SERVER ${offsite_host} &
		done
		touch ${PW_FOLDER}/shutdown.offsite
	fi
}

SHUTDOWN_SERVERS(){
	# TEST=true
	echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${1} servers" >> ${PW_LOGFILE}
	[ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE}
	
	if [ "${1^^}" == "MAIN" ]; then
		if [ ! -f ${PW_FOLDER}/shutdown.main ]; then
			touch ${PW_FOLDER}/shutdown.main
			
			echo -en "Disabling CLS in vCenter ... "
			if [ "${TEST}" != "true" ]; then
				run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
				run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
				run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
			fi
			echo "Done"
		
			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
				echo -en "Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
				echo "Done"
			done
			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
				echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
				echo "Done"
			done
			# sleep 1m
			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
				echo -en "Verifying iSCSI-PRI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
				echo "Done"
			done
			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
				echo -en "Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				if [ "${TEST}" != "true" ]; then
					run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
					run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
				fi
				echo "Done"
			done

			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
				echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
				[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
				echo -e "${idsCL[Green]}Continuing in background\n"
			done
			
		fi
	
	elif [ "${1^^}" == "SYS" ]; then
		[ ! -f ${PW_FOLDER}/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN
		if [ ! -f ${PW_FOLDER}/shutdown.sys ]; then
			touch ${PW_FOLDER}/shutdown.sys
			
			if [ "${1^^}" == "SYS" ]; then
				echo -en "Disabling CLS in vCenter ... "
				if [ "${TEST}" != "true" ]; then
					run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
					run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
					run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
				fi
				echo "Done"
			fi
						
			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
				echo -en "Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
				echo "Done"
			done
			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
				echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
				echo "Done"
			done
			# sleep 1m
			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
				echo -en "Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
				echo "Done"
			done
			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
				echo -en "Shutting Down VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				if [ "${TEST}" != "true" ]; then
					${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
					${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
				fi
				echo "Done"
			done

			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
				echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
				[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
				echo -e "${idsCL[Green]}Continuing in background\n"
			done
			
		fi
		
	elif [ "${1^^}" == "ALL" ]; then
		if [ ! -f ${PW_FOLDER}/shutdown.all ]; then
			touch ${PW_FOLDER}/shutdown.all
			
			echo -en "Disabling CLS in vCenter ... "
			if [ "${TEST}" != "true" ]; then
				run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
				run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
				run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
			fi
			echo "Done"
						
			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
				echo -en "Stopping all iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
				echo "Done"
			done
			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
				echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
				echo "Done"
			done
			# sleep 1m
			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
				echo -en "Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
				echo "Done"
			done
			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
				echo -en "Shutting Down VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				if [ "${TEST}" != "true" ]; then
					${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
					${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
				fi
				echo "Done"
			done
			
			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do		
				echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
				[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
				echo -e "${idsCL[Green]}Continuing in background\n"
			done
			
		fi
		
	elif [ "${1^^}" == "OFFSITE" ]; then
		if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
			touch ${PW_FOLDER}/shutdown.offsite
			
			echo -en "Disabling CLS in vCenter ... "
			if [ "${TEST}" != "true" ]; then
				run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
				run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
				run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
			fi
			echo "Done"
			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
				echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
				echo "Done"
			done
			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
				echo -en "Stopping VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
				echo "Done"
			done
			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
				echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
				[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
				echo -e "${idsCL[Green]}Continuing in background\n"
			done
			
		fi
	fi
}
MAINT_MODE_VERIFY_SHUTDOWN(){
	ESXIHOST=${1}
	MaintMode=""
	until [ "${MaintMode}" == "Enabled" ]; do
		sleep 3s
		MaintMode=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get')
	done
	ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"'
	ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
	
	if [ "${ESXIHOST}" == "10.10.2.10" ]; then
		while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done
		 "$(CHECK_HOST ${ESXIHOST})" == "false" ] && snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2
	fi
}
SHUTDOWN_SERVER(){
	# TEST=true
	if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
		tmp=${PW_iDRACHOST_SHORTNAMES[${1}]}
		ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]}
	elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
		ESXIHOST=${1}
	elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
		ESXIHOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${1}]}
	elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
		tmp=${PW_HOST_IDRACS[${1}]}
		ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]}
	else
		ESXIHOST=''
	fi	
	echo -e "\n${idsCL[Yellow]}ESXi Host Powerdown"
	DIVIDER . yellow 75
	
	
	if [ "${ESXIHOST}" != "" ]; then
		host=${PW_ESXI_HOST_NAMES[${ESXIHOST}]}
		c=0; spcA=''; spc1=`expr 21 - ${#host} - 1`; until [ ${c} = ${spc1} ]; do spcA="${spcA} "; c=`expr ${c} + 1`; done
		c=0; spcB=''; spc1=`expr 18 - ${#ESXIHOST} - 1`; until [ ${c} = ${spc1} ]; do spcB="${spcB} "; c=`expr ${c} + 1`; done
		echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${host}${spcA}"
		echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${ESXIHOST}${spcB}"
		echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${PW_HOST_IDRACS[${host}]}${idsCL[Default]}"
		DIVIDER . yellow 75
		echo	
	
		echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${host} (${ESXIHOST})" >> ${PW_LOGFILE}
		
		if [ "${ESXIHOST}" == "10.10.2.10" ]; then
			echo -e "${idsCL[LightYellow]}Since this is the iSCSI-PRI host server, all iSCSI-PRI VM's are being shutdown ... "
			for host in "${PW_MAINSITE_HOSTS[@]}"; do
				echo -en "${idsCL[LightCyan]}Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}] ... "
				[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
				echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
			done
			echo -e "${idsCL[LightYellow]}All iSCSI-PRI VM's have been sent shutdown signals\n"
			sleep 10s
			
		elif [ "${ESXIHOST}" == "10.10.2.17" ]; then
			echo -e "${idsCL[LightYellow]}Since this is the iSCSI-SYS host server, all iSCSI-SYS VM's are being shutdown ... "
			for host in "${PW_MAINSITE_HOSTS[@]}"; do
				echo -en "${idsCL[LightCyan]}Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}] ... "
				[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
				echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
			done
			echo -e "${idsCL[LightYellow]}All iSCSI-SYS VM's have been sent shutdown signals\n"
			sleep 10s
		fi
		
		echo -en "${idsCL[LightCyan]}Enabling maintenance mode & starting iSCSI VM migrations ... "
		[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
		[ "${TEST}" != "true" ] && sleep 25s
		echo -e "${idsCL[Green]}Done\n"
	
		echo -en "${idsCL[LightCyan]}Shutting down remaining VM's ... "
		if [ "${TEST}" != "true" ]; then
			run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
			if [ "${ESXIHOST}" == "10.10.2.10" ] || [ "${ESXIHOST}" == "10.10.2.17" ]; then
				[ "${ESXIHOST}" == "10.10.2.10" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
				[ "${ESXIHOST}" == "10.10.2.17" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
				run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
			fi
			run=$(${PW_FOLDER}/esxi-scripts/cls-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
		fi
		echo -e "${idsCL[Green]}Done\n"
		
		echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
		[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
		echo -e "${idsCL[Green]}Continuing in background\n"
		
		if [ "${ESXIHOST}" == "10.10.2.10" ]; then
			echo -en "${idsCL[LightCyan]}Waiting for Host ${host} (${ESXIHOST}) to shutdown ... "
			[ "${TEST}" != "true" ] && while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done
			echo -e "${idsCL[Green]}Done\n"
			
			echo -en "${idsCL[LightCyan]}Powering off SAS Enclosure ... "
			[ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2)
			echo -e "${idsCL[Green]}Done\n"
			
		else
			echo -e "${idsCL[LightCyan]}The host ${host} (${ESXIHOST}) will be shutdown momentarily ... \n"
		fi
		
	else
		echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
	fi
	echo
}

ISCSIVMSHUTDOWN(){
	for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
		echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}' that are running on an iSCSI-PRI drive ... "
		${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
		echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
		echo 
	done
	
	echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..."
	echo
}
ISCSISYSVMSHUTDOWN(){
	for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
		echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}' that are running on an iSCSI-SYS drive ... "
		${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
		echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
		echo 
	done
	
	echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..."
	echo
}

HOSTPOWER(){
	echo -e "\n${idsCL[Yellow]}Host Power Control"
	DIVIDER . yellow 75
		
	if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
		idracip=${1}
	elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
		tmp=${PW_ESXI_HOST_NAMES[${1}]}
		idracip=${PW_HOST_IDRACS[${tmp}]}
	elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
		idracip=${PW_HOST_IDRACS[iDS-vMS-Host${1}]}
	elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
		idracip=${PW_HOST_IDRACS[${1}]}
	else
		idracip=''
	fi
	
	if [ "${idracip}" != "" ]; then
		host=${PW_iDRACHOST_SHORTNAMES[${idracip}]}
		c=0; spcA=''; spc1=`expr 21 - ${#host} - 1`; until [ ${c} = ${spc1} ]; do spcA="${spcA} "; c=`expr ${c} + 1`; done
		c=0; spcB=''; spc1=`expr 18 - ${#PW_HOST_ADDRESSES[${host}]} - 1`; until [ ${c} = ${spc1} ]; do spcB="${spcB} "; c=`expr ${c} + 1`; done
		echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${host}${spcA}"
		echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${PW_HOST_ADDRESSES[${host}]}${spcB}"
		echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${idracip}${idsCL[Default]}"
		DIVIDER . yellow 75
		echo
		
		if [ "${2,,}" == "on" ] || [ "${2,,}" == "off" ] || [ "${2,,}" == "status" ]; then
			if [ "${idracip}" == "10.10.2.20" ] && [ "${2,,}" == "on" ]; then
				echo -en "${idsCL[LightCyan]}Powering ON the SAS Enclosure ... "
				[ "${TEST}" != "true" ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 1 >/dev/null 2>&1)
				echo -e "${idsCL[Green]}Done\n"
			fi
			
			echo -e "${idsCL[Green]}"
			ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power ${2,,}
			echo -e "${idsCL[Default]}"
			
			if [ "${idracip}" == "10.10.2.20" ] && [ "${2,,}" == "off" ]; then
				echo -en "${idsCL[LightCyan]}Powering OFF the SAS Enclosure ... "
				[ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
				echo -e "${idsCL[Green]}Done\n"
			fi
		else
			echo -e "${idsCL[LightRed]}'${2,,}' command is not recognized${idsCL[Default]}"
		fi
	else
		echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
	fi
	echo -e "\n"
}

LOGROTATE(){
	STOPALL_SERVICES
	lm=`expr $(date +'%m') - 1`
	if [ ${lm} = 0 ]; then
		lm=12
		ly=`expr $(date +'%Y') - 1`
	else
		ly=$(date +'%Y')
	fi
	[ ${lm} -lt 10 ] && lm="0${lm}"
	ARCHIVEFOLDER=${PW_LOGARCHIVE}/${ly}-${lm}
	mkdir -p ${ARCHIVEFOLDER}
	sleep 5s
	mv ${PW_LOGFOLDER}/log* ${ARCHIVEFOLDER}/ 2> /dev/null
	mv ${PW_LOGFOLDER}/service.* ${ARCHIVEFOLDER}/ 2> /dev/null
	# source ${PW_FOLDER}/defaults.inc
	STARTALL_SERVICES
	backupstokeep=(`ls -d ${PW_LOGARCHIVE}/* 2> /dev/null | sort | tail -12`)
	for folder in $(ls -d ${PW_LOGARCHIVE}/* | sort -r); do
		keep=0
		for filetokeep in ${backupstokeep[@]}; do
			if [ "${folder}" == "${filetokeep}" ]; then
				keep=1; break
			fi
		done;
		[ ${keep} == 0 ] && rm -Rf ${folder}
	done
}

GET_SNAPSHOTS(){
	echo
	for SENSOR in ${PW_SENSOR_ORD[@]}; do
		if [ "${PW_HOST_ADDRESSES[${SENSOR}]}" != "" ]; then
			ESXIHOST=${PW_HOST_ADDRESSES[${SENSOR}]}
			echo -e "${idsCL[LightGreen]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]} (${ESXIHOST}) "
			DIVIDER . lightGreen 70
			echo -en "${idsCL[LighGreen]}Looking for snapshots...${idsCL[Default]}"
			snapshots=`${PW_FOLDER}/esxi-scripts/get-snapshots.ps1 ${ESXIHOST}`
			if [ "${snapshots}" != "" ]; then
				echo -en "\033[2K\033[1A"
				echo -e "${snapshots}"
				# for snapshot in $(echo ${snapshots} | jq -r '.[] | @base64'); do
				# 	vm=`echo ${snapshot} | base64 --decode | jq -r '.VM'`
				# 	name=`echo ${snapshot} | base64 --decode | jq -r '.Name'`
				# 	created=`echo ${snapshot} | base64 --decode | jq -r '.Created'`
				# 	size=`echo ${snapshot} | base64 --decode | jq -r '.SizeGB'`
				# 	echo "$vm - $name - ${c}reated - $size"
				# done
				
			else
				echo -e "\033[2K\e[0K\r No snapshots found"
			fi
			DIVIDER true lightGreen 70
		fi
	done
}

TEST(){
	start=`date +%s`
	temptotal=0
	daysback=${1}
	logsfound=0;
	while IFS=$'\t' read date sensorid temp hum ;do
		# echo date:$date sensorid:$sensorid temp:$temp hum:$hum
		temptotal=$(bc <<< "scale=2; ${temptotal}+${temp}")
		# echo "- ${temptotal} + ${temp}"
		((logsfound++))
		
	done  < <(${mysql_conn} -se "USE servermonitor; SELECT date,sensorid,temp,hum from sensor_data WHERE sensorid=2 AND date BETWEEN 
        DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
		
	average=$(bc <<< "scale=2; ${temptotal}/${logsfound}")
	
	echo "Logs found: ${logsfound}"
	echo "Average: ${average}'F"
	echo
	
	# QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('$(date +'%Y-%m-%d %H:%M:%S')','${INVSTATSINFO}')"
	# ${mysql_conn} -e "USE servermonitor; ${QRY}"
	
	end=`date +%s`
	runtime=$((end-start))
	echo "runtime: ${runtime}"
	echo
}



#########################################################
if [[ "${1}" != *"_service" ]] && [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
	echo
	DIVIDER . lightGreen 70
	msg="| PowerWall Management ${idsCL[Default]}                 ${idsCL[DarkGray]}(ver-${VERS})"
	c=0; spc=''; spc1=`expr 81 - ${#msg}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
	echo -e "${idsCL[LightGreen]}${msg}${spc}${idsCL[LightGreen]}|${idsCL[Default]}"
	DIVIDER . lightGreen 70
fi
# if [ ${action-x} ]; then
	case ${action} in
		services)
			[ "${2}" == "stop" ] && STOPALL_SERVICES
			[ "${2}" == "start" ] && STARTALL_SERVICES
			[ "${2}" == "restart" ] && RESTARTALL_SERVICES
			[ "${2}" == "status" ] && CHECK_SERVICES
		;;
		checktemp) CHECKTEMP ${2};;
		dailytemp)
			start=`date +%s`
			DAILYTEMP
			# SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)"
			end=`date +%s`; runtime=$((end-start))
			echo -e "\nruntime: ${runtime}\n"
			;;
		checkpower) CHECKPOWER ${2};;
		test) CHECKTEMP ${2} ${3};;
		iscsi-vm-shutdown) 
			[ "${2^^}" == "SYS" ] && ISCSISYSVMSHUTDOWN || ISCSIVMSHUTDOWN
			;;
		shutdownhost) SHUTDOWN_SERVER ${2};;
		shutdown_servers) SHUTDOWN_SERVERS ${2} ${3};;
		hostpower) HOSTPOWER ${2} ${3};;
		
		get-snapshots) GET_SNAPSHOTS;;
		
		service)
			if [ "${2}" = "all" ]; then
				if [ "${3}" == "stop" ]; then
					STOPALL_SERVICES
				elif [ "${3}" == "start" ]; then
					STARTALL_SERVICES
				elif [ "${3}" == "restart" ]; then
					RESTARTALL_SERVICES
				elif [ "${3}" == "status" ]; then
					CHECK_SERVICES
				fi
			else
				if [ "${3}" = "start" ] || [ "${3}" = "" ]; then
					if [ "${2}" = "temp" ]; then
						# SENDNOTICE "Temp Monitor Startup" "$(DAILYTEMP)" &
						CHECKTEMP_SERVICE
					elif [ "${2}" = "power" ]; then
						CHECKPOWER_SERVICE
					elif [ "${2}" = "powerlogger" ]; then
						POWERLOGGER_SERVICE
					fi
				
				elif [ "${3}" = "stop" ]; then
					if [ "${2}" = "temp" ]; then
						STOP_SERVICE temp
						for SENSOR in ${!PW_SENSOR_ID[@]}; do
							logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
							echo "(`date +'%Y-%m-%d %H:%M:%S'`) - ${SENSOR} - Service Stopped" >> ${logtemp}
						done
					
					elif [ "${2}" = "power" ]; then
						STOP_SERVICE power
						echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${PW_LOGPOWER}
					
					elif [ "${2}" = "powerlogger" ]; then
						STOP_SERVICE powerlogger
					
					fi
				
				elif [ "${3}" = "restart" ]; then
					SERVICE_MNG ${2} stop
					slep 2s
					SERVICE_MNG ${2} start
					
				elif [ "${3}" = "status" ]; then
					srvc=${PW_POWERWALL_SERVICES[${2}]}
					c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
					echo -e "${idsCL[LightCyan]}${srvc}${spc}: "
					[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
					echo -e "${idsCL[Default]}"
					echo
				fi
			fi
		;;
		
		logrotate) LOGROTATE;;
		update)
			# echo -en "${idsCL[LightCyan]}Will reboot monitoring services in the background to apply updates ... "
			# RESTARTALL_SERVICES >/dev/null 2>&1 &
			# echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
		;;
		status)
			CHECKPOWER
			CHECKTEMP
			CHECK_SERVICES
		;;
		average) AVERAGETEMP ${2} ${3} ${4};;
		*)
			echo -e "${idsCL[White]} Usage: ${idsCL[LightCyan]}powerwall {option}"
			echo
			echo -e "    ${idsCL[Yellow]}status                     ${idsCL[Default]}--> PowerWall Status Information"
			echo -e "    ${idsCL[Yellow]}checktemp {searchstring}   ${idsCL[Default]}--> Check Temperature Sensors"
			echo -e "    ${idsCL[Yellow]}checkpower {avg}           ${idsCL[Default]}--> Check Power Readings"
			echo -e "    ${idsCL[LightYellow]}  avg = { noavg='Does not display average' / "
			echo -e "                  #='Num of days for avg' }${idsCL[Default]} --> Defaults to 3 days avg"
			echo
			echo -e "    ${idsCL[Yellow]}shutdownhost {host}        ${idsCL[Default]}--> Shutdown ESXI host and its VMs"
			echo -e "    ${idsCL[LightYellow]}  host = {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}"
			echo -e "    ${idsCL[Yellow]}iscsi-vm-shutdown {type}   ${idsCL[Default]}--> Shutdown VM's running on iSCSI-(PRI or SYS) Datastore(s)"
			echo -e "    ${idsCL[LightYellow]}  type = { sys='iSCSI-SYS' / "
			echo -e "               main='iSCSI-MAIN' }${idsCL[Default]} --> Defaults to main (iSCSI-PRI)"
			echo -e "    ${idsCL[Yellow]}shutdown_servers {type}    ${idsCL[Default]}--> Shutdown selection of servers"
			echo -e "    ${idsCL[LightYellow]}  type = { main / sys / all }"
			echo -e "    ${idsCL[Yellow]}hostpower {host}           ${idsCL[Default]}--> Control host chassis power"
			echo -e "    ${idsCL[LightYellow]}  host = {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}"
			echo
			# echo
			# echo -e "    ${idsCL[Yellow]}services {cmd}             ${idsCL[Default]}--> Manage all monitor services at once"
			# echo -e "    ${idsCL[LightYellow]}  cmd = {start/stop/restart/status}"
			# echo
			echo -e "    ${idsCL[Yellow]}service {srvc} {cmd}       ${idsCL[Default]}--> Manage an individual monitor service"
			echo -e "    ${idsCL[LightYellow]}  srvc = { all / temp / power / powerlogger }"
			echo -e "    ${idsCL[LightYellow]}  cmd = { start / stop / restart / status }"
			echo
			echo -e "    ${idsCL[Yellow]}update                     ${idsCL[Default]}--> Check for and install new PowerWall Script updates"
			echo
		;;
	esac
# fi

exit 0