#!/usr/bin/env bash
# powerwall - CLI commands to control VM guest power

action="${1}"

source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/powerwall/defaults.inc
source ${FOLDER}/powerwall.conf

TEST=true

shopt -s lastpipe

VERBOSE=false


CHECKTEMP(){
	cw=20;
	echo
	echo -e "${idsCL[Yellow]}Sensor Temperature(s)                            ${idsCL[Green]}normal${idsCL[Default]} /${idsCL[Yellow]}warning${idsCL[Default]}/ ${idsCL[Red]}critical${idsCL[Default]}"
	DIVIDER . yellow 75
	if [ "${1}" != "" ]; then
		# reading=$(CHECKTEMPSENSOR ${1})
		# c=0; spc=''; spc1=`expr ${cw} - ${#1}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
		# echo -e "${idsCL[LightCyan]}${1}${spc}${idsCL[Default]}: ${reading}"
		
		for SENSOR in ${!SENSOR_ID[@]}; do
			if [[ "${SENSOR}" = *"${1}"* ]]; then
				reading=$(CHECKTEMPSENSOR ${SENSOR})
				c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
				echo -e "${idsCL[LightCyan]}${SENSOR}${spc}${idsCL[Default]}: ${reading}"
			fi
		done
		
	else
		for SENSOR in ${!SENSOR_ID[@]}; do
			reading=$(CHECKTEMPSENSOR ${SENSOR})
			c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
			echo -e "${idsCL[LightCyan]}${SENSOR}${spc}${idsCL[Default]}: ${reading}"
		done
	fi
	echo
}
CHECKTEMPSENSOR(){
	echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
	echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
	
	if [ "${SENSOR_TYPE[${1}]}" == "mqtt" ]; then
		mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
		echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
		
	elif [ "${SENSOR_TYPE[${1}]}" == "system" ]; then
		temp_c=$(vcgencmd measure_temp)
		temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
		
	fi
	
	if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
		temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
		#echo "$temp_c -> $temp_f"

		if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
			echo -en "${idsCL[Yellow]}WARNING  ($temp_f'F)${idsCL[Default]}"
		elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then
			echo -en "${idsCL[Red]}CRITICAL ($temp_f'F)${idsCL[Default]}"
		else
			echo -en "${idsCL[Green]}Normal   ($temp_f'F)${idsCL[Default]}"
		fi
		
		c=0; spc=''; spc1=`expr 13 - ${#temp_f} - 2`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
		[ ${#temp_warn} == 2 ] && spcw='  ' || spcw=' '
		[ ${#temp_crit} == 2 ] && spcc='  ' || spcc=' '
		echo -e "${spc}[${spcw}${idsCL[Green]}<=$(expr ${temp_warn} - 1)'F${idsCL[Default]} /${spcw}${idsCL[Yellow]}${temp_warn}'F${idsCL[Default]} /${spcc}${idsCL[Red]}${temp_crit}'F${idsCL[Default]} ]"
	fi
}

CHECKTEMP_SERVICE(){
	cw=20
	declare -A last_temp
	while true; do
		
		for SENSOR in ${!SENSOR_ID[@]}; do
			logtemp=${LOGFOLDER}/log-temp-${SENSOR}
			[ ! -f ${logtemp} ] && touch ${logtemp}
			errtime=$(expr `date +%s` - $(stat -c %Y ${logtemp}))
			[ $errtime -ge 3600 ] && relog=1 || relog=0
			
			if [ "${SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then
				mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
				echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
				echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
				
			elif [ "${SENSOR_TYPE[${SENSOR}]}" == "system" ]; then
				temp_c=$(vcgencmd measure_temp)
				temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
				temp_h=''
			fi
			datetime=`date +'%Y-%m-%d %H:%M:%S'`
			
			if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
				temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
	
				echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
				echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
				echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
			
				if [ "${last_temp[${SENSOR}]}" != "" ]; then
					temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
					[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
				else
					last_temp[${SENSOR}]=0
					temp_diff=0
				fi
					
				if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
					if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
						SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
						echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
						last_temp[${SENSOR}]=$temp_f
					fi
					echo "($datetime) - ${temp_f}F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
				
				elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
					echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
					if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
						SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
						echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
						last_temp[${SENSOR}]=$temp_f
						SHUTDOWN_MAIN ${SENSOR}
						SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1
						echo -e "($datetime) - ${SENSOR} - ${temp_f}F - Main servers have been shutdown" >> ${logtemp}

					fi
				elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
					echo "($datetime) - ${temp_f}F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
					if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
						SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down system servers!!" 1
						echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
						last_temp[${SENSOR}]=$temp_f
						SHUTDOWN_SYS ${SENSOR} TEMP
						SHUTDOWN_CRIT ${SENSOR}
						SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1
						echo -e "($datetime) - ${SENSOR} - ${temp_f}F - System servers have been shutdown" >> ${logtemp}
					fi
				else
					if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
						SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
						echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
						echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
						[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
					elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
						SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
NORMAL TEMP: ${temp_f}'F"
						echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
						[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
					fi
					last_temp[${SENSOR}]=1
					[ $relog -eq 1 ] && echo "($datetime) - ${SENSOR} - ${temp_f}F - Normal Temp" >> ${logtemp}
					[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
				fi
				if [ "${temp_h}" != "" ]; then
					QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
				else
					QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
				fi
				${mysql_conn} -e "${QRY}"
			else
				SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1
				echo "($datetime) - ${SENSOR} - alert sent" >> ${logtemp}
			fi
			
		done
		
		sleep 10
		
	done # & 
}

CHECKPOWER(){
	JSONSTATS=$(curl -s "http://10.10.0.61/stats.json")
	INPUTV=`echo ${JSONSTATS} | jq '.inputs .inV'`
	INPUTA=`echo ${JSONSTATS} | jq '.inputs .inA'`
	OUTPUTV=`echo ${JSONSTATS} | jq '.outputs .outV'`
	OUTPUTA=`echo ${JSONSTATS} | jq '.outputs .outA'`
	BATTV=`echo ${JSONSTATS} | jq '.inputs .battV'`
	BATTA=`echo ${JSONSTATS} | jq '.inputs .xfA'`
	
	OUTPUTV2=$(awk -vp_val="$OUTPUTV" 'BEGIN{print p_val/2}')
	OUTPUTA2=$(awk -vp_val="$OUTPUTA" 'BEGIN{print p_val*2}')
	
	echo
	echo -e "${idsCL[Yellow]}Power Information"
	DIVIDER . yellow 75
		
	echo -e "${idsCL[LightCyan]}Input Power         ${idsCL[Default]}: ${idsCL[Green]}${INPUTV}V / ${INPUTA}A"
	echo -e "${idsCL[LightCyan]}Output Power        ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTV}V (${OUTPUTV2}V) / ${OUTPUTA}A (${OUTPUTA2}A @ ${OUTPUTV2}V)"
	echo -e "${idsCL[LightCyan]}Battery Power       ${idsCL[Default]}: ${idsCL[Green]}${BATTV}V / ${BATTA}A ${idsCL[Default]}"
	[ "${action}" != "" ] && echo
}
		
CHECKACV_SERVICE(){
	voltstatus=0
	battstatus=0
	while true; do
		INPUTACV=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .inV')
		BATTVOLT=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .battV')
		datetime=`date +'%Y-%m-%d %H:%M:%S'`
		if [ ${INPUTACV} -ge ${min_acvolt} ]; then
			if [ $voltstatus -eq 3 ]; then
				echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${logacv}
				SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected
VOLTAGE: ${INPUTACV}V"
				echo "($datetime) alert sent" >> ${logacv}
				battstatus=0
			else
				errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
				[ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${logacv}
				if [ $voltstatus -eq 0 ]; then
					SENDNOTICE "Power Nominal" "($datetime) Service Startup
Normal voltage detected
VOLTAGE: ${INPUTACV}V
BATT VOLTAGE: ${BATTVOLT}V"
					echo "($datetime) - Input: ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${logacv}
					echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${logacv}
				fi
			fi
			voltstatus=1
		else
			echo "($datetime) - ${INPUTACV}V - LOW Voltage" >> ${logacv}
			if [ $voltstatus -lt 3 ]; then
				SENDNOTICE "POWER ALERT - LOW POWER!!" "($datetime) Power off or low voltage detected
VOLTAGE: ${INPUTACV}V
BATT VOLTAGE: ${BATTVOLT}V" 1
				echo "($datetime) alert sent" >> ${logacv}
				last_battvolt=$BATTVOLT
			fi
			voltstatus=3
			if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
				echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
				if [ $battstatus -lt 2 ]; then
					echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logacv}
					SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V
Shutting down main servers" 1
					SHUTDOWN_MAIN server
					battstatus=2
				fi
				
			elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
				echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
				if [ $battstatus -lt 3 ]; then
					echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logacv}
					SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V
Shutting down all servers" 1
					SHUTDOWN_SYS server
					
					battstatus=3
				fi
				
			else
				errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
				[ $errtime -ge 3600 ] && echo "($datetime) - ${BATTVOLT}V - Battery Voltage" >> ${logacv}
				
				battstatus=1
				
			fi
			volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*}))
			[ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1))
			if [ $volt_diff -gt 0 ]; then
				SENDNOTICE "BATTERY VOLTAGE CHANGE" "($datetime) Battery Voltage: ${BATTVOLT}V"
				echo -e "($datetime) - ${BATTVOLT}V - Battery Voltage Change" >> ${logacv}
				last_battvolt=$BATTVOLT
			fi
		fi
		
	    sleep 10
	done # & 
}

POWERLOGGER_SERVICE(){
	while true; do
		datetime=`date +'%Y-%m-%d %H:%M:%S'`
		
		JSONSTATS=$(curl -s "http://10.10.0.61/stats.json")
		inV=`echo ${JSONSTATS} | jq '.inputs .inV'`
		inA=`echo ${JSONSTATS} | jq '.inputs .inA'`
		outV=`echo ${JSONSTATS} | jq '.outputs .outV'`
		outA=`echo ${JSONSTATS} | jq '.outputs .outA'`
		battV=`echo ${JSONSTATS} | jq '.inputs .battV'`
		xfA=`echo ${JSONSTATS} | jq '.inputs .xfA'`
		
		QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
		${mysql_conn} -e "USE servermonitor; ${QRY}"
		QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
		${mysql_conn} -e "USE servermonitor; ${QRY}"
		QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
		${mysql_conn} -e "USE servermonitor; ${QRY}"
		QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${JSONSTATS}')"
		${mysql_conn} -e "USE servermonitor; ${QRY}"
				
		sleep 1m
		
	done
}

RESTART_SERVICES(){
	STOPALL_SERVICES
	sleep 5s
	STARTALL_SERVICES
}
STOPALL_SERVICES(){
	for srvc in "${POWERWALL_SERVICES[@]}"; do
		echo "Stopping $srvc"
		/bin/systemctl stop $srvc
	done
}
STARTALL_SERVICES(){
	for srvc in "${POWERWALL_SERVICES[@]}"; do
		echo "Starting $srvc"
		/bin/systemctl start $srvc
	done
}

STOP_SERVICE(){
	echo "stop: ${1}"
	service_pid=`systemctl show --property MainPID --value ${POWERWALL_SERVICES[${1}]}`
	/usr/bin/pkill -P $service_pid
}

SHUTDOWN_MAIN(){
	SHUTDOWN_SERVERS MAIN
	echo
}
SHUTDOWN_SYS(){
	SHUTDOWN_SERVERS SYS
	curl --data "cmd=PWRoff" http://10.10.0.61/cmd
	echo
}
SHUTDOWN_CRIT(){
	if [ "${1}" == "RaspberryPI-CPU" ]; then
		SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) Powerwall System shutting down" 1
		echo -e "($datetime) - ${SENSOR} - CRITICAL - Powerwall System shutting down" >> ${logtemp}
		/sbin/poweroff
	fi
}

SHUTDOWN_SERVERS(){
	
	# HOSTS=${1}HOSTS[@]
	# for ESXIHOST in "${!HOSTS}"; do
	# 	echo "Shutting Down: ${ESXIHOST}"
	# 	/usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
	# 	/usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh"
	# 	/usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
	# 	/usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
	# done

		
	if [ "${1}" = "MAIN" ]; then
		touch $FOLDER/shutdown.main
		echo -en "Disabling CLS in vCenter ... "
		if [ $TEST != true ]; then
			ssh root@${VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1
			ssh root@${VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
			ssh root@${VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
		fi
		echo "Done"
		
		for ESXIHOST in "${ESXIHOSTS[@]}"; do
			echo -en "Stopping iSCSI VM's on: ${ESXIHOST} ... "
			[ $TEST != true ] && ${FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
			echo "Done"
		done
		for ESXIHOST in "${ESXIHOSTS[@]}"; do
			echo -en "Verifying iSCSI VM's stopped on: ${ESXIHOST} ... "
			[ $TEST != true ] && ${FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1
			echo "Done"
		done
		# sleep 1m
		for ESXIHOST in "${MAINHOSTS[@]}"; do
			echo -en "Stopping Remaining VM's on: ${ESXIHOST} ... "
			if [ $TEST != true ]; then
				ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0' &
				${FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
				${FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
			fi
			echo "Done"
		done
		sleep 1m
		for ESXIHOST in "${MAINHOSTS[@]}"; do
			echo -en "Shutting down host: ${ESXIHOST} ..."
			if [ $TEST != true ]; then
				ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"'
				ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
			fi
			echo "Done"
		done
		
		for ESXIHOST in "${MAINHOSTS[@]}"; do
			echo -en "Waiting for Host (${ESXIHOST}) to shutdown... "
			if [ $TEST != true ]; then
			    while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
			    	sleep 1
			    done
			fi
			echo "Done"
		done
		
	elif [ "${1}" = "SYS" ]; then
		(([ ! -f $FOLDER/shutdown.main ] && [ "${2}" == "TEMP" ]) || [ $TEST = true ]) && SHUTDOWN_SERVERS MAIN
		[ $TEST != true ] && touch $FOLDER/shutdown.sys

		for ESXIHOST in "${SYSHOSTS[@]}"; do
			echo -en "Suspending DB, PiHole & vCenter VM's on: ${ESXIHOST} ... "
			[ $TEST != true ] && ${FOLDER}/esxi-scripts/sys-suspend.ps1 ${ESXIHOST} >/dev/null 2>&1
			echo "Done"
		done
		for ESXIHOST in "${SYSHOSTS[@]}"; do
			echo -en "Shutting Down VM's on: ${ESXIHOST} ... "
			if [ $TEST != true ]; then
				${FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
				${FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
			fi
			echo "Done"
		done
		for ESXIHOST in "${SYSHOSTS[@]}"; do
			echo -en "Shutting Down Host: ${ESXIHOST} ... "
			if [ $TEST != true ]; then
				ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0'
				ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' &
				ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
			fi
			echo "Done"
		done

		for ESXIHOST in "${SYSHOSTS[@]}"; do
			echo -en "Waiting for Host (${ESXIHOST}) to shutdown ... "
			if [ $TEST != true ]; then
				while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
					sleep 1
				done
			fi
			echo "Done"
		done
			
	fi
}
SHUTDOWN_SERVER(){
	echo -en "Enabling Maintenance mode and will wait for iSCSI VM's to begin migrating ... "
	[ $TEST != true ] && ${FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXI_HOST_NAMES[${1}]} & >/dev/null 2>&1
	sleep 45s
	echo "Done"
	
	echo -en "Shutting Down VM's on: ${1} ... "
	if [ $TEST != true ]; then
		${FOLDER}/esxi-scripts/vm-shutdown.ps1 ${1} >/dev/null 2>&1
		${FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${1} >/dev/null 2>&1
	fi
	echo "Done"

	echo -en "Shutting Down Host: ${1} ... "
	if [ $TEST != true ]; then
		ssh root@${1} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' & >/dev/null 2>&1
		ssh root@${1} 'esxcli system maintenanceMode set -e false -t 0' >/dev/null 2>&1
	fi
	echo "Done"

	echo -en "Waiting for Host (${1}) to shutdown ... "
	if [ $TEST != true ]; then
	    while ping -qw 10 -c3 "${1}">/dev/null 2>&1; do
	    	sleep 1
	    done
	fi
	echo "Done"
}

ISCSIVMSHUTDOWN(){
	for ESXIHOST in "${ESXIHOSTS[@]}"; do
		echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}, that are running on an iSCSI drive ... "
		${FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
		echo -e "${idsCL[Green]}DONE${idsCL[Default]}"
		echo 
	done
	
}

TEST(){
	# echo "Getting info from ${1}"
	# ${FOLDER}/esxi-scripts/sys-iscsi-shutdown.ps1 ${1}
	
	for ESXIHOST in "${ESXIHOSTS[@]}"; do
		ssh-copy-id root@${ESXIHOST}
	done
	
}


if [[ "${1}" != *"_service" ]] && [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
	echo
	DIVIDER . lightGreen 50
	c=0; spc=''; spc1=`expr 18 - ${#VERS}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
	echo -e "${idsCL[LightGreen]}| PowerWall Management ${idsCL[Default]}  ${idsCL[DarkGray]}(ver-${VERS})${spc}${idsCL[LightGreen]}|${idsCL[Default]}"
	DIVIDER . lightGreen 50
fi

# if [ ${action-x} ]; then
	case $action in
		stopall) STOPALL_SERVICES;;
		startall) STARTALL_SERVICES;;
		restartall) RESTART_SERVICES;;
		checktemp) CHECKTEMP ${2};;
		checkpower) CHECKPOWER;;
		test) TEST ${2};;
		iscsi-vm-shutdown) ISCSIVMSHUTDOWN;;
		
		shutdownhost) SHUTDOWN_SERVER ${2};;
		shutdown_servers) SHUTDOWN_SERVERS ${2};;
		
		checktemp_service) 
			if [ "${2}" = "stop" ]; then
				STOP_SERVICE Temp
				for SENSOR in ${!SENSOR_ID[@]}; do
					logtemp=${LOGFOLDER}/log-temp-${SENSOR}
					echo "(`date +'%Y-%m-%d %H:%M:%S'`) - ${SENSOR} - Service Stopped" >> ${logtemp}
				done
			else
				CHECKTEMP_SERVICE
			fi
		;;
		checkacv_service) 
			if [ "${2}" = "stop" ]; then
				STOP_SERVICE ACV
				echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${logacv}
			else
				CHECKACV_SERVICE
			fi
		;;
		powerlogger_service) 
			if [ "${2}" = "stop" ]; then
				STOP_SERVICE powerlogger
			else
				POWERLOGGER_SERVICE
			fi
		;;
		update);;
		*)
			CHECKPOWER
			CHECKTEMP
		;;
	esac
# fi

exit 0