#!/usr/bin/env bash
# powerwall - CLI commands to control VM guest power

action="$1"

source /opt/idssys/powerwall/defaults.inc
source /opt/idssys/powerwall/powerwall.conf
source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc

shopt -s lastpipe

VERBOSE=false


#ssh root@10.5.10.35 '/vmfs/volumes/NFS_ESXi-Vault/esxi-shutdown.sh > /dev/null 2>&1' & > /dev/null 2>&1

CHECKTEMP(){
	[ "${1}" != "" ] && SENSORS=("${1}") || SENSORS=${SENSOR_ID}
	for x in "${!SENSOR_ID[@]}"; do printf "[%s]=%s\n" "$x" "${SENSOR_ID[$x]}" ; done
	for x in "${!SENSORS[@]}"; do printf "[%s]=%s\n" "$x" "${SENSORS[$x]}" ; done

	for SENSOR in ${!SENSORS[@]}; do
		echo "HERE: ${SENSORS[${SENSOR}]}"
	done
	
	# mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
	# echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
	# if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
	# 	temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
	# 	#echo "$temp_c -> $temp_f"
	#
	# 	echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
	# 	echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
	#
	# 	if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
	# 		echo "WARNING: $temp_f°F"
	# 	elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then
	# 		echo "CRITICAL: $temp_f°F"
	# 	else
	# 		echo "normal: $temp_f°F"
	# 	fi
	# fi
}

CHECKTEMP_SERVICE(){
	logsrvc=${FOLDER}/log-${1}
	[ ! -f ${logsrvc} ] && touch ${logsrvc}
	last_temp=0
	mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
	while true; do
	    ${mqtt_conn} -t tele/${1}/SENSOR | while read -r mqtt_message;   do
			echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
			echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
			echo $mqtt_message | cut -d'T' -f 2 | cut -d"\"" -f 3 | read temp_d
			echo $mqtt_message | cut -d'T' -f 3 | cut -d"\"" -f 1 | read temp_t
			# datetime="${temp_d} ${temp_t}"
			# datetime=$(TZ=UTC0 date -d "${datetime}" +%s)
			# datetime=`date -d @${datetime} +'%Y-%m-%d %H:%M:%S'`
			datetime=`date +'%Y-%m-%d %H:%M:%S'`
			
			if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
				temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
				
				#echo "$temp_c -> $temp_f"
		
				echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
				echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
				echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 3 | read temp_crit_sys
				
				temp_diff=$((${last_temp%.*} - ${temp_f%.*}))
				[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
						
				if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
					if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
						SENDNOTICE "${1} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)" 1
						echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc}
						last_temp=$temp_f
					fi
					echo "($datetime) - ${temp_f}F - WARNING TEMP - (Difference of ${temp_diff}°)" >> ${logsrvc}
					
				elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
					echo "($datetime) - ${temp_f} F - CRITICAL - (Difference of ${temp_diff} F)" >> ${logsrvc}
					if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
						SENDNOTICE "${1} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down servers!!" 1
						echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc}
						last_temp=$temp_f
						# SHUTDOWN_SERVERS MAIN
						SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1
						echo -e "($datetime) - ${temp_f}F - Main servers have been shutdown" >> ${logsrvc}

					fi
				elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
					echo "($datetime) - ${temp_f}F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logsrvc}
					if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
						SENDNOTICE "${1} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down system servers!!" 1
						echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc}
						last_temp=$temp_f
						# SHUTDOWN_SERVERS SYS
						SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1
						echo -e "($datetime) - ${temp_f}F - System servers have been shutdown" >> ${logsrvc}
					fi
				else
					if [ $last_temp -gt 1 ]; then
						SENDNOTICE "${1} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp}°F"
						echo -e "($datetime) - ${temp_f}F - Back to NORMAL TEMP - Previous Temp: ${last_temp}F" >> ${logsrvc}
						echo "($datetime) alert sent" >> ${logsrvc}
					elif [ $last_temp -eq 0 ]; then
						SENDNOTICE "${1} TEMP NORMAL" "($datetime) Service Startup\n NORMAL TEMP: ${temp_f}°F"
						echo -e "($datetime) - ${temp_f}F - Service Startup - NORMAL TEMP" >> ${logsrvc}
					fi
					last_temp=1
					errtime=$(expr `date +%s` - $(stat -c %Y ${logsrvc}))
					[ $errtime -ge 3600 ] && echo "($datetime) - ${temp_f}F - Normal Temp" >> ${logsrvc}
				fi
				
				QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[$1]}','${datetime}','${temp_f}','${temp_h}')"
				${mysql_conn} -e "${QRY}"
			else
				SENDNOTICE "${1} ERROR" "($datetime) ERROR reading temperature" 1
				echo "($datetime) alert sent" >> ${logsrvc}
			fi
	    done
	    sleep 10
	done # & 
}

CHECKPOWER(){
	JSONSTATS=$(curl -s "http://10.10.0.61/stats.json")
	INPUTV=`echo ${JSONSTATS} | jq '.inputs .inV'`
	INPUTA=`echo ${JSONSTATS} | jq '.inputs .inA'`
	OUTPUTV=`echo ${JSONSTATS} | jq '.outputs .outV'`
	OUTPUTA=`echo ${JSONSTATS} | jq '.outputs .outA'`
	BATTV=`echo ${JSONSTATS} | jq '.inputs .battV'`
	BATTA=`echo ${JSONSTATS} | jq '.inputs .xfA'`
	
	OUTPUTV2=$(awk -vp_val="$OUTPUTV" 'BEGIN{print p_val/2}')
	OUTPUTA2=$(awk -vp_val="$OUTPUTA" 'BEGIN{print p_val*2}')
		
	echo "Input Power: ${INPUTV}V / ${INPUTA}A"
	echo "Output Power: ${OUTPUTV}V (${OUTPUTV2}V) / ${OUTPUTA}A (${OUTPUTA2}A @ ${OUTPUTV2}V)"
	echo "Battery Power: ${BATTV}V / ${BATTA}A"
}
		
CHECKACV_SERVICE(){
	voltstatus=0
	battstatus=0
	while true; do
		INPUTACV=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .inV')
		BATTVOLT=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .battV')
		datetime=`date +'%Y-%m-%d %H:%M:%S'`
		if [ ${INPUTACV} -ge ${min_acvolt} ]; then
			if [ $voltstatus -eq 3 ]; then
				echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${logacv}
				SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected\nVOLTAGE: ${INPUTACV}V"
				echo "($datetime) alert sent" >> ${logacv}
				battstatus=0
			else
				errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
				[ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${logacv}
				if [ $voltstatus -eq 0 ]; then
					SENDNOTICE "Power Nominal" "($datetime) Service Startup\nNormal voltage detected\nVOLTAGE: ${INPUTACV}V"
					echo "($datetime) - ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${logacv}
				fi
			fi
			voltstatus=1
		else
			echo "($datetime) - ${INPUTACV}V - LOW Voltage" >> ${logacv}
			if [ $voltstatus -lt 3 ]; then
				SENDNOTICE "POWER ALERT - LOW POWER!!" "($datetime) Power off or low voltage detected\nVOLTAGE: ${INPUTACV}V\nBATT VOLTAGE: ${BATTVOLT}V" 1
				echo "($datetime) alert sent" >> ${logacv}
				last_battvolt=$BATTVOLT
			fi
			voltstatus=3
			if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
				echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
				if [ $battstatus -lt 2 ]; then
					echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logacv}
					SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V\nShutting down main servers" 1
					# SHUTDOWN_SERVERS MAIN
					battstatus=2
				fi
				
			elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
				echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
				if [ $battstatus -lt 3 ]; then
					echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logacv}
					SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V\nShutting down all servers" 1
					# SHUTDOWN_SERVERS SYS
					curl --data "cmd=PWRoff" http://10.10.0.61/cmd
					
					battstatus=3
				fi
				
			else
				errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
				[ $errtime -ge 3600 ] && echo "($datetime) - ${BATTVOLT}V - Battery Voltage" >> ${logacv}
				
				battstatus=1
				
			fi
			volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*}))
			[ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1))
			if [ $volt_diff -gt 0 ]; then
				SENDNOTICE "BATTERY VOLTAGE CHANGE" "($datetime) Battery Voltage: ${BATTVOLT}V"
				echo -e "($datetime) - ${BATTVOLT}V - Battery Voltage Change" >> ${logacv}
				last_battvolt=$BATTVOLT
			fi
		fi
		
	    sleep 10
	done # & 
}

POWERLOGGER_SERVICE(){
	while true; do
		datetime=`date +'%Y-%m-%d %H:%M:%S'`
		
		JSONSTATS=$(curl -s "http://10.10.0.61/stats.json")
		inV=`echo ${JSONSTATS} | jq '.inputs .inV'`
		inA=`echo ${JSONSTATS} | jq '.inputs .inA'`
		outV=`echo ${JSONSTATS} | jq '.outputs .outV'`
		outA=`echo ${JSONSTATS} | jq '.outputs .outA'`
		battV=`echo ${JSONSTATS} | jq '.inputs .battV'`
		xfA=`echo ${JSONSTATS} | jq '.inputs .xfA'`
		
		QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
		${mysql_conn} -e "USE servermonitor; ${QRY}"
		QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
		${mysql_conn} -e "USE servermonitor; ${QRY}"
		QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
		${mysql_conn} -e "USE servermonitor; ${QRY}"
		QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${JSONSTATS}')"
		${mysql_conn} -e "USE servermonitor; ${QRY}"
				
		sleep 1m
		
	done
}

RESTART_SERVICES(){
	STOPALL_SERVICES
	sleep 5s
	STARTALL_SERVICES
}
STOPALL_SERVICES(){
	for srvc in "${POWERWALL_SERVICES[@]}"; do
		echo "Stopping $srvc"
		/bin/systemctl stop $srvc
	done
}
STARTALL_SERVICES(){
	for srvc in "${POWERWALL_SERVICES[@]}"; do
		echo "Starting $srvc"
		/bin/systemctl start $srvc
	done
}

STOP_SERVICE(){
	echo "stop: ${1}"
	service_pid=`systemctl show --property MainPID --value ${POWERWALL_SERVICES[${1}]}`
	/usr/bin/pkill -P $service_pid
}

SHUTDOWN_SERVERS(){
	touch $FOLDER/shutdown
	
	HOSTS=${1}HOSTS[@]	
	for ESXIHOST in "${!HOSTS}"; do
		echo "Shutting Down: ${ESXIHOST}"
		/usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
		/usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh"
		/usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
		/usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
	done
	for ESXIHOST in "${!HOSTS}"; do
		echo -en "Waiting for Host (${ESXIHOST}) to shutdown... "
	    while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
	    	sleep 1
	    done
		echo "Ok"
	done
		
	if [ "${1}" = "MAIN" ]; then
		/opt/idssys/powerwall/esxi-scripts/sys-iscsi-shutdown.ps1 10.10.2.17 >/dev/null 2>&1
		
		/usr/bin/ssh root@10.10.2.10 "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
		echo -en "Waiting for iSCSI-Server to shutdown... "
	    while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
	    	sleep 1
	    done
		echo "Ok"
	fi
}
SHUTDOWN_SERVER(){	
	echo "Shutting Down: ${1}"
	/usr/bin/ssh root@${1} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${1}.sh"
	/usr/bin/ssh root@${1} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${1}.sh"
	/usr/bin/ssh root@${1} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${1}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${1}.sh"
	/usr/bin/ssh root@${1} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${1}.sh"
	
	echo -en "Waiting for Host (${1}) to shutdown... "
    while ping -qw 10 -c3 "${1}">/dev/null 2>&1; do
    	sleep 1
    done
	echo "Ok"
}

ISCSIVMSHUTDOWN(){
	for ESXIHOST in "${ESXIHOSTS[@]}"; do
		echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}, that are running on an iSCSI drive ... "
		# /opt/idssys/powerwall/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
		echo -e "${idsCL[Green]}DONE${idsCL[Default]}"
		echo 
	done
	
}

TEST(){
	echo "Getting info from ${1}"
	/opt/idssys/powerwall/esxi-scripts/sys-iscsi-shutdown.ps1 ${1}
	
}




if [ ${action-x} ]; then
	case $action in
		stopall) STOPALL_SERVICES;;
		startall) STARTALL_SERVICES;;
		restartall) RESTART_SERVICES;;
		checktemp) CHECKTEMP ${2};;
		checkpower) CHECKPOWER;;
		test) TEST ${2};;
		iscsi-vm-shutdown) ISCSIVMSHUTDOWN;;
		
		shutdownhost) SHUTDOWN_SERVER ${2};;
		
		checktemp_service) 
			if [ "${2}" = "stop" ]; then
				STOP_SERVICE ${3}
				echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${FOLDER}/log-${3}
			else
				CHECKTEMP_SERVICE ${3}
			fi
		;;
		checkacv_service) 
			if [ "${2}" = "stop" ]; then
				STOP_SERVICE ACV
				echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${logacv}
			else
				CHECKACV_SERVICE
			fi
		;;
		powerlogger_service) 
			if [ "${2}" = "stop" ]; then
				STOP_SERVICE powerlogger
			else
				POWERLOGGER_SERVICE
			fi
		;;
	esac
fi

exit 0