#!/usr/bin/env bash
# NodeMgmt - Galera/NGINX Node Management Scripts

action="$1"

if [ ! -f /opt/idssys/settings/nodemgmt.conf ]; then
	if [ ! -d /opt/idssys/settings ]; then
		mkdir /opt/idssys/settings
	fi
	mv /opt/idssys/nodemgmt/settings.conf /opt/idssys/settings/nodemgmt.conf
fi

source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/settings/nodemgmt.conf
source /opt/idssys/nodemgmt/defaults.inc

#source /opt/idssys/nodemgmt/ssl-check.sh


# ========================================================= #

STATUS(){
	start=`date +%s`
	ST_ACTION=${1}
	if [ "${ST_ACTION}" = "report" ]; then
		unset idsCL idsBG idsST
		idsCL=('')
		idsBG=('')
		idsST=('')
	fi
	declare -i cw; declare -i spc1; declare -i c
	########################
	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ]; then
		if [ "${ST_ACTION}" != "check" ]; then
			echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}"
			echo
		fi
		if [ ! -z ${LOCAL_SERVICES+x} ];
			then NHCMD='ssh root@10.5.10.51'
			else NHCMD=''
		fi
		for rcheck in "${REPL_CHECK[@]}"; do
			${NHCMD} rm -f ${FOLDER}/test.repl
			daterun=`date +%Y-%m-%d-%H-%M-%S`
			if [ "${NHCMD}" = "" ]; then
				echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
				yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
			else
				${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
				yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
			fi
		done
		if [ "${ST_ACTION}" = "repl" ]; then
			sleep 5
		fi
	fi
	########################
	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then
		if [ "${ST_ACTION}" != "check" ]; then
			echo -e "${idsST[Bold]}"; DIVIDER
			echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}"
			DIVIDER; echo -e "${idsST[Reset]}"
		fi
		if [ ! -z ${LOCAL_SERVICES+x} ]; then
			if [ "${ST_ACTION}" != "check" ]; then
				lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
				uptime=`uptime -p`
				echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
				DIVIDER false green
			fi
			for srvc in "${LOCAL_SERVICES[@]}"; do
				if [ "${ST_ACTION}" != "check" ]; then
					c=0; cw=22; spc=''
					spc1=${cw}-${#NM_SERVICES[${srvc}]}
					until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
					echo -en " ${NM_SERVICES[${srvc}]}$spc: "
				fi
				
				if [ "$(systemctl is-active ${srvc})" != "active" ]; then
					if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
						if [ "${ST_ACTION}" != "check" ]; then
							echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
						fi
						echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}
						touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
					else
						errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
						if [ $errtime -gt ${RENOTIFY} ]; then
							if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
								mv ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
							fi
							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
							if [ "${ST_ACTION}" != "check" ]; then
								echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
							fi
							echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}
							touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
						fi
					fi
				else
					if [ "${ST_ACTION}" != "check" ]; then
						echo -e "${idsCL[Green]}Running${idsCL[Default]}"
					fi
					if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
						if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
						else
							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
						fi
						echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}
						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
					fi
				fi
			done
			
			if [ "${ST_ACTION}" != "check" ]; then echo; fi
		fi
		########################
		nid=1
		for nip in "${NODE_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
				else NCMD="ssh root@${nip}"; LH=''
			fi
			if [ "${ST_ACTION}" != "check" ]; then
				uptime=`${NCMD} uptime -p`
				echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
				DIVIDER false green
			fi
			if [ "${NCMD}" != "" ]; then
				checkhost=$(CHECK_HOST ${nip})
			fi
			if [ "${checkhost}" != "false" ]; then
				if [ -f ${FOLDER}/${nip}.down ]; then
					if [ -f ${FOLDER}/${nip}.errtime ]; then
						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
					else
						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
					fi
					rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
					echo "iDS-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
				fi
				for srvc in "${NODE_SERVICES[@]}"; do
					if [ "${ST_ACTION}" != "check" ]; then
						c=0; cw=22; spc=''
						spc1=${cw}-${#NM_SERVICES[${srvc}]}
						until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
						echo -en " ${NM_SERVICES[${srvc}]}$spc: "
					fi
					
					if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
						if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
							if [ "${ST_ACTION}" != "check" ]; then
								echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
							fi
							echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
							touch ${FOLDER}/${nip}-${srvc}.down
						else
							errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
							if [ $errtime -gt ${RENOTIFY} ]; then
								if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
									mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
								fi
								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
								if [ "${ST_ACTION}" != "check" ]; then
									echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
								fi
								echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
								touch ${FOLDER}/${nip}-${srvc}.down
							fi
						fi
					else
						if [ "${ST_ACTION}" != "check" ]; then
							echo -e "${idsCL[Green]}Running${idsCL[Default]}"
						fi
						if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
							if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
							else
								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
							fi
							echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
							rm -f ${FOLDER}/${nip}-${srvc}.down
							rm -f ${FOLDER}/${nip}-${srvc}.errtime
						fi
					fi
				done
			else
				errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
				if [ $errtime -gt ${RENOTIFY} ]; then
					if [ ! -f ${FOLDER}/${nip}.errtime ]; then
						mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
					fi
					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
					if [ "${ST_ACTION}" != "check" ]; then
						echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
					fi
					echo "iDS-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
				else
					if [ "${ST_ACTION}" != "check" ]; then
						echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
					fi
				fi
				touch ${FOLDER}/${nip}.down
			fi
			if [ "${ST_ACTION}" != "check" ]; then echo; fi
			nid=`expr $nid + 1`
		done
	fi
	########################
	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then
		if [ "${ST_ACTION}" != "check" ]; then
			echo -e "${idsST[Bold]}"; DIVIDER
			echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
			DIVIDER; echo -e "${idsST[Reset]}"
		fi
		nid=1
		for nip in "${NODE_HOSTS[@]}"; do
			if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
				if [ "${ST_ACTION}" != "check" ]; then
					echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
					DIVIDER false green
				fi
				checkhost=$(CHECK_HOST ${nip})
				if [ "${checkhost}" != "false" ]; then
					if [ -f ${FOLDER}/${nip}.down ]; then
						rm -f ${FOLDER}/${nip}.down
						echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
					fi
					for rcheck in "${REPL_CHECK[@]}"; do
						if [ "${ST_ACTION}" != "check" ]; then
							c=0; cw=22; spc=''
							spc1=${cw}-${#REPL_DESC[${rcheck}]}
							timeout=`date --date='20 seconds' +%s`
							until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
							echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
						fi
						checked=false
						until [ "${checked}" = "" ]; do
							if [ "${NHCMD}" = "" ]; then
								checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
							else
								checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
							fi
							if [ "`date +%s`" -gt "$timeout" ]; then
								timeout=true
								break
							fi
						done
						if [ "${timeout}" = "true" ]; then
							if [ "${ST_ACTION}" != "check" ]; then
								echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
							fi
							if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
								touch ${FOLDER}/${nip}-${rcheck}.down
								echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
							fi
						else
							if [ "${ST_ACTION}" != "check" ]; then
								echo -e "${idsCL[Green]}Good${idsCL[Default]}"
							fi
							if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
								rm -f ${FOLDER}/${nip}-${rcheck}.down
								echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
							fi
						fi
						
					done
				else
					if [ "${ST_ACTION}" != "check" ]; then
						echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
					fi
					if [ ! -f ${FOLDER}/${nip}.down ]; then
						touch ${FOLDER}/${nip}.down
						echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
					fi
				fi
				if [ "${ST_ACTION}" != "check" ]; then echo; fi
			fi
			nid=`expr $nid + 1`
		done
	fi
	
	if [ "${ST_ACTION}" != "check" ]; then
		echo ""
		if [ -z $action ] || [ "${action}" = "gui" ]; then
			DIVIDER true
			ENTER2CONTINUE
		fi
	fi
	end=`date +%s`
	runtime=$((end-start))
	echo "runtime: ${runtime}"
}

# STATUS-CHECK(){
# 	start=`date +%s`
# 	if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
# 		if [ ! -z ${LOCAL_SERVICES+x} ]; then NHCMD='ssh root@10.5.10.51'
# 		else NHCMD=''; fi
# 		# for rcheck in "${REPL_CHECK[@]}"; do
# 		# 	${NHCMD} rm -f ${FOLDER}/test.repl
# 		# 	daterun=`date +%Y-%m-%d-%H-%M-%S`
# 		# 	if [ "${NHCMD}" = "" ]; then
# 		# 		echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
# 		# 		yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
# 		# 	else
# 		# 		${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
# 		# 		yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
# 		# 	fi
# 		# done
# 		if [ ! -z ${LOCAL_SERVICES+x} ]; then
# 			lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
# 			for srvc in "${LOCAL_SERVICES[@]}"; do
# 				if [ "$(systemctl is-active ${srvc})" != "active" ]; then
# 					if [ ! -f ${FOLDER}/localhost-${srvc}.down ]; then
# 						echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
# 						touch ${FOLDER}/localhost-${srvc}.down
# 					fi
# 				elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then
# 					echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
# 					rm -f ${FOLDER}/localhost-${srvc}.down
# 					rm -f ${FOLDER}/localhost-${srvc}.errtime
# 				fi
# 			done
# 		fi
# 		if [ -z ${LOCAL_SERVICES+x} ] || [ "${1}" = "all" ]; then
# 			nid=1
# 			for nip in "${NODE_HOSTS[@]}"; do
# 				checkhost=$(CHECK_HOST ${nip})
# 				if [ "${checkhost}" != "false" ]; then
# 					if [ -f ${FOLDER}/${nip}.down ]; then
# 						rm -f ${FOLDER}/${nip}.down
# 						echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
# 					fi
# 					if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
# 						else NCMD="ssh root@${nip}"
# 					fi
# 					for srvc in "${NODE_SERVICES[@]}"; do
# 						if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
# 							if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
# 								echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# 								touch ${FOLDER}/${nip}-${srvc}.down
# 							else
# 								errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
# 								if [ $errtime -gt ${RENOTIFY} ]; then
# 									if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
# 										mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
# 									fi
# 									toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
# 									echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# 									touch ${FOLDER}/${nip}-${srvc}.down
# 								fi
# 							fi
# 						elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
# 							echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# 							rm -f ${FOLDER}/${nip}-${srvc}.down
# 							rm -f ${FOLDER}/${nip}-${srvc}.errtime
# 						fi
# 					done
# 				elif [ ! -f ${FOLDER}/${nip}.down ]; then
# 					touch ${FOLDER}/${nip}.down
# 					echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
# 				fi
# 				nid=`expr $nid + 1`
# 			done
# 			##REPLICATION CHECK
# 			# nid=1
# 			# for nip in "${NODE_HOSTS[@]}"; do
# 			# 	if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
# 			# 	if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
# 			# 		checkhost=$(CHECK_HOST ${nip})
# 			# 		if [ "${checkhost}" != "false" ]; then
# 			# 			if [ -f ${FOLDER}/${nip}.down ]; then
# 			# 				rm -f ${FOLDER}/${nip}.down
# 			# 				echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
# 			# 			fi
# 			# 			for rcheck in "${REPL_CHECK[@]}"; do
# 			# 				timeout=`date --date='20 seconds' +%s`
# 			# 				checked=false
# 			# 				until [ "${checked}" = "" ]; do
# 			# 					if [ "${NHCMD}" = "" ]; then
# 			# 						checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
# 			# 					else
# 			# 						checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
# 			# 					fi
# 			# 					if [ "`date +%s`" -gt "$timeout" ]; then
# 			# 						timeout=true
# 			# 						break
# 			# 					fi
# 			# 				done
# 			# 				if [ "${timeout}" != "true" ]; then
# 			# 					if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
# 			# 						rm -f ${FOLDER}/${nip}-${rcheck}.down
# 			# 						echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
# 			# 					fi
# 			# 				elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
# 			# 					touch ${FOLDER}/${nip}-${rcheck}.down
# 			# 					echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
# 			# 				fi
# 			# 			done
# 			# 		elif [ ! -f ${FOLDER}/${nip}.down ]; then
# 			# 			touch ${FOLDER}/${nip}.down
# 			# 			echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
# 			# 		fi
# 			# 	fi
# 			# 	nid=`expr $nid + 1`
# 			# done
# 		fi
# 	fi
# 	end=`date +%s`
# 	runtime=$((end-start))
# 	echo "runtime: ${runtime}"
# }

NEWCERT(){
	echo ""
	if [ -z ${1+x} ]; then
		echo -e -n "${idsCL[LightCyan]}Create certificate for what address: ${idsCL[Default]}"
		read NEW_CERT
		echo ""
	else
		NEW_CERT=${1}
	fi
	if [[ ${NEW_CERT} == *","* ]]; then
		IFS=','; NEW_CERTS=(${NEW_CERT}); unset IFS
		MAIN_CERT=${NEW_CERTS[0]}
	else
		MAIN_CERT=${NEW_CERT}
	fi
	if [ "${NEW_CERT}" != "" ]; then
		echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}"
		echo ""
		
		for nip in "${NODE_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then 
				`ssh root@${nip} service nginx stop`
			fi
		done
		
		# $CERT_DAEMON certonly --webroot -w /var/www/html -d ${NEW_CERT}
		$CERT_DAEMON certonly --dry-run --webroot -w /var/www/html -d ${NEW_CERT}
		if [ -f /etc/letsencrypt/live/${MAIN_CERT}/cert.pem ]; then
			chown -R root:letsencrypt /etc/letsencrypt
			chmod -R 6775 /etc/letsencrypt
			if [ -f /etc/nginx/sites-available/${MAIN_CERT} ]; then
				rm -f  ${FOLDER}/cert-request.lastrun
				daterun=`date +%Y-%m-%d-%H-%M-%S`
				echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun
				yes | cp -rfH ${FOLDER}/cert-request.lastrun /etc/letsencrypt/cert-request.lastrun
				DIVIDER true
				echo -e "Since this certificate is tied to a site in NGINX, we will reload NGINX across the nodes."
				echo
				echo -en "${idsCL[LightYellow]}Waiting for certificate replication across the nodes... ${idsCL[Default]}"
				checked=false
				timeout=`date --date='2 minutes' +%s`
				until [ "${checked}" = "" ]; do
					tchecked=''
					for nip in "${NODE_HOSTS[@]}"; do
						if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
							else NCMD="ssh root@${nip}"
						fi
						if [ "${NCMD}" != "" ]; then
							tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-request.lastrun" | diff - /etc/letsencrypt/cert-request.lastrun`
						fi
					done
					checked=${tchecked}
					if [ "`date +%s`" -gt "$timeout" ]; then
						echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
						timeout=true
						echo "Timeout occurred in waiting for replication between nodes. (${NEW_CERT})" | mail -s "Cert-Request" ${STATUS_CHECK_EMAIL}
						break
					fi
				done
				if [ "${timeout}" != "true" ]; then
					echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
				fi
				echo
				SERVICE nginx reload
				for nip in "${NODE_HOSTS[@]}"; do
					if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then 
						`ssh root@${nip} service nginx start`
					fi
				done
			fi
			echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}"
		else
			echo ""
			echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}"
		fi
		echo ""
		if [ -z $action ] || [ "${action}" = "gui" ]; then
			DIVIDER true
			ENTER2CONTINUE
		fi
	else
		echo "You havent entered a site address."
		sleep 1
		NEWCERT
		exit 0
	fi
}

CERTRENEW(){
	echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}"
	echo
	$CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
	chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
	chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
	yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
	daterun=`date +%Y-%m-%d-%H-%M-%S`
	echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
	DIVIDER true
	echo -en "${idsCL[LightYellow]}Waiting for certificate replication between the nodes... ${idsCL[Default]}" 
	checked=false
	timeout=`date --date='2 minutes' +%s`
	until [ "${checked}" = "" ]; do
		tchecked=''
		for nip in "${NODE_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
				else NCMD="ssh root@${nip}"
			fi
			if [ "${NCMD}" != "" ]; then
				tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-renewal.lastrun" | diff - /etc/letsencrypt/cert-renewal.lastrun`
			fi
		done
		checked=${tchecked}
		if [ "`date +%s`" -gt "$timeout" ]; then
			echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
			timeout=true
			if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
				echo "Timeout occurred in waiting for replication between nodes." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
			fi
			break
		fi
	done
	if [ "${timeout}" != "true" ]; then
		echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
	fi
	echo
	SERVICE nginx reload 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
	echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}"
	echo ""
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
		ENTER2CONTINUE
	fi
}
NIGHTLYRENEW(){
	rm -f ${FOLDER}/cert-renewal.lastrun
	$CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun
	chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
	chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
	yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
	daterun=`date +%Y-%m-%d-%H-%M-%S`
	echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
	checked=false
	timeout=`date --date='2 minutes' +%s`
	until [ "${checked}" = "" ]; do
		tchecked=''
		for nip in "${NODE_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
				else NCMD="ssh root@${nip}"
			fi
			if [ "${NCMD}" != "" ]; then
				tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-renewal.lastrun" | diff - /etc/letsencrypt/cert-renewal.lastrun`
			fi
		done
		checked=${tchecked}
		if [ "`date +%s`" -gt "$timeout" ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then
			echo "Timeout occurred in waiting for replication between nodes." | mail -s "Nightly-Cert-Renewal" ${STATUS_CHECK_EMAIL}
			break
		fi
	done
	SERVICE nginx reload &>> ${FOLDER}/cert-renewal.lastrun
}

LISTCERTS(){
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
	fi
	echo 
	echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}"
	DIVIDER
	echo -e "Subject Name                  Expires       Alternate Subjects"
	DIVIDER
	AWK=$(which awk)
	DATE=$(which date)
	GREP=$(which grep)
	OPENSSL=$(which openssl)
	SED=$(which sed)
	for certdir in /etc/letsencrypt/live/*/ ; do
		SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
		SUBJECTNAMES=$(openssl x509 -in ${certdir}/cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV)
		SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, }
		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g")
		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g")
		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g")
		CERTEXPIRE=$(date -d "$(: | openssl x509 -in ${certdir}/cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s');
		DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS)));

		declare -i cw; declare -i spc1; declare -i c
		c=0; cw=30; spc=''
		spc1=${cw}-${#SUBJECT}
		until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
		if [ $DUEIN -gt $CERTEXPIRE ]; then
			date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}"
		else
			date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}"
		fi
		echo -e "${idsCL[Cyan]}${SUBJECT}${spc}${date}    ${idsCL[Cyan]}${SUBJECTNAMES}${idsCL[Default]}" 
	done
	echo 
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
		ENTER2CONTINUE
	fi
	echo ""
}

CHECK-CERTS(){
	rm -f /opt/idssys/nodemgmt/ssl-cert-check.lastrun
	/opt/idssys/nodemgmt/ssl-cert-check/ssl-cert-check -a -f /opt/idssys/nodemgmt/ssl-cert-check/ssldomains -e ${STATUS_CHECK_EMAIL} > /opt/idssys/nodemgmt/ssl-cert-check.lastrun
}

BACKUP(){
	BACKUP_FOLDER=/opt/idssys/backups/node-backup
	#BACKUP_FOLDER=/opt/idssys/backups/node-backups/${NODE_HOSTNAME} #/`date +%Y-%m-%d`
	#BACKUP_TIMEFOLDER=${BACKUP_DAYFOLDER}/`date +%H-%M`
	if [ "${action}" != "backup" ]; then
		echo -e "${idsCL[Green]}Backing up Node Settings and Files...${idsCL[Default]}"
		DIVIDER true
	fi
	if [ ! -d ${BACKUP_FOLDER} ]; then
		mkdir -p ${BACKUP_FOLDER}
	fi
	declare -A BACKUP_ITEMS
	BACKUP_ITEMS[nginx-settings]=/etc/nginx
	BACKUP_ITEMS[letsencrypt-certs]=/etc/letsencrypt
	BACKUP_ITEMS[gitea]=/var/lib/gitea
	BACKUP_ITEMS[nginx-logs]='/var/www/!NGINX-Logs'
	#BACKUP_ITEMS[webserver-files]=/var/www
	for item in "${!BACKUP_ITEMS[@]}"
	do
		if [ "${action}" != "backup" ]; then
			echo -en "Backing up '${item}'... "
		fi
		#tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${BACKUP_ITEMS[$item]} .
		tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -g ${BACKUP_FOLDER}/backup.snar  -C ${BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git'
		if [ "${action}" != "backup" ]; then
			echo -e "${idsCL[Green]}Ok${idsCL[Default]}"
		fi
	done
	touch ${BACKUP_FOLDER}/backup.lastrun
	if [ "${action}" != "backup" ]; then
		echo ""
		DIVIDER true
	fi
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		ENTER2CONTINUE
	fi
}

DELSITE(){
	while [ $# -gt 0 ]; do
	    case "$1" in
	        -site) DEL_SITE=${2};;
	        -ssl) DEL_SSL=${2};;
			-list) DELSITES; exit 0;;
	   	 	-*)
				echo "Invalid option: '${1}' requires an argument" 1>&2
				echo ""
				echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
				width=33
				printf "%-${width}s- %s\n" "    -site {FQDN address}" "(*required)"
				printf "%-${width}s- %s\n" "    -ssl {yes or [no]}" "Delete SSL certs as well"
				printf "%-${width}s- %s\n" "    -list" "List sites (same as running nodemgmt delsites)"
				echo "}"
				exit 1;;
	    esac
		shift
	done
	if [ -z ${DEL_SITE+x} ]; then
		echo -e -n "${idsCL[LightCyan]}Delete what site address: ${idsCL[Default]}"
		read DEL_SITE
		echo ""
	fi
	if [[ $DEL_SSL =~ ^[Nn]$ ]]; then
		DEL_SSL=no
	elif [[ $DEL_SSL =~ ^[Yy]$ ]]; then
		DEL_SSL=yes
	elif [ -z ${DEL_SSL+x} ]; then
		echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${DEL_SITE}' as well? [y/N]${idsCL[Default]} "
		read DEL_SSL
	fi
	if [ ! -z ${DEL_SITE+x} ] && [ "${DEL_SITE}" != "" ]; then
		echo -e "${idsCL[LightRed]}Deleting site '${idsCL[Red]}${DEL_SITE}${idsCL[LightRed]}'...${idsCL[Default]}"
		echo ""
	
		echo -e "${idsCL[LightRed]}[[Removing Files and Folders]]${idsCL[Default]}"
		echo -e "${idsCL[LightRed]}-------------------------------------------${idsCL[Default]}"

		nid=1
		for nip in "${NODE_HOSTS[@]}"
		do
			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Removing from iDS-Node${nid} ($nip)... ${idsCL[Default]}"
			#if [ -f /etc/nginx/sites-available/${DEL_SITE} ]; then
				${NCMD} rm -f /etc/nginx/sites-available/${DEL_SITE}
				${NCMD} rm -f /etc/nginx/sites-enabled/${DEL_SITE}
				#fi
			#if [ -d /var/www/${DEL_SITE} ]; then
				${NCMD} rm -rf /var/www/${DEL_SITE}
				#fi
			if [ "${DEL_SSL}" = "yes" ]; then
				${NCMD} rm -rf /etc/letsencrypt/archive/${DEL_SITE}
				${NCMD} rm -rf /etc/letsencrypt/live/${DEL_SITE}
				${NCMD} rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf
			fi
			echo -e "${idsCL[Green]}OK${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
		echo ""
		#SERVICE nginx reload
		echo -e "${idsCL[LightRed]}Site has been deleted.${idsCL[Default]}"
	else
		echo "Missing arguments"
		echo ""
		echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
		width=33
		printf "%-${width}s- %s\n" "    -site {FQDN address}" "Site to delete"
		printf "%-${width}s- %s\n" "    -ssl {yes or [no]}" "Delete SSL certs as well"
		printf "%-${width}s- %s\n" "    -list" "List sites (same as running nodemgmt delsites)"
		echo "}"
		exit 1
	fi
}

DELSITES(){
	echo
	echo -e "${idsCL[Red]}Select a site to delete...${idsCL[Default]}"
	DIVIDER true
	sid=1
	for siteconf in /etc/nginx/sites-available/* ; do
		[ -e "$siteconf" ] || continue
		IFS='/'; site_conf=(${siteconf}); unset IFS
		SITES[${sid}]=${site_conf[4]}
		sid=`expr $sid + 1`
	done
	for s in "${!SITES[@]}"; do
		echo " [${s}] ${SITES[${s}]}"
	done
	echo ""
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		echo " [B] Back"
	fi
	echo " [Q] Quit"
	echo ""
	echo -e -n "${idsCL[LightYellow]}Please select a site from above from above:${idsCL[Default]} "
	read selsite
	echo ""
	if [ -z ${SITES[$selsite]} ] && [ "${selsite}" != "Q" ] && [ "${selsite}" != "q" ] && [ "${selsite}" != "B" ] && [ "${selsite}" != "b" ]; then
		echo "Thats an invaild option,"
		echo "please select a valid option only."
		sleep 1
		DELSITES
		exit 0
	elif [ "${selsite}" = "Q" ] || [ "${selsite}" = "q" ]; then
		exit 0
	elif [ "${selsite}" = "B" ] || [ "${selsite}" = "b" ]; then
		GUI
	else
		while :
		do
			echo -e -n "${idsCL[LightRed]}Are you sure you want to delete '${idsST[Bold]}${idsCL[Red]}${SITES[${selsite}]}${idsST[Reset]}${idsCL[LightRed]}'? [y/N]${idsCL[Default]} "
			read response
			echo ""
			if [[ $response =~ ^[Yy]$ ]]; then
				echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${SITES[${selsite}]}' as well? [y/N]${idsCL[Default]} "
				read sslresponse
				DELSITE -site ${SITES[${selsite}]} -ssl ${sslresponse}
				echo ""
				DIVIDER
				ENTER2CONTINUE
				break
			else
				break
			fi			
		done
		DELSITES
		exit 0
	fi
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		ENTER2CONTINUE
	fi
}

NEWSITE(){
	echo
	while [ $# -gt 0 ]; do
		case "$1" in
			-site) NEW_SITE=${2};;
			-type) SITE_TYPE=${2};;
			-ssl) CREATE_SSL=${2};;
			-proxy_scheme) PROXYSCHEME=${2};;
			-proxy_host) PROXYHOST=${2};;
			-proxy_port) PROXYPORT=${2};;
			-h | -help | --help)
				echo "Missing arguments"
				echo ""
				echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {"
				width=33
				printf "%-${width}s- %s\n" "    -site {FQDN address(,es)}" "(new site and aliases, comma separated)"
				printf "%-${width}s- %s\n" "    -ssl {yes or no}" "(defaults to yes)"
				printf "%-${width}s- %s\n" "    -type {'local' or 'proxy'}" "(defaults to local)"
				printf "%-${width}s- %s\n" "    -port {host port}" "(proxy backend host)"
				printf "%-${width}s- %s\n" "    -host {IP or FQDN}" "(proxy backend port)"
				printf "%-${width}s- %s\n" "    -scheme {http or https}" "(proxy backend scheme)"
				echo "}"
				exit 1;;
		esac
		shift
	done
		
	#if [ -z ${SITE_TYPE+x} ]; then SITE_TYPE=local; fi
	#if [ -z ${CREATE_SSL+x} ]; then CREATE_SSL=true; fi
	if [ -z ${NEW_SITE+x} ]; then
		echo -e -n "${idsCL[LightCyan]}New site domain name: ${idsCL[Default]}"
		read NEW_SITE
		showdivide=yes
	fi
	if [ -z ${CREATE_SSL+x} ]; then
		echo -e -n "${idsCL[LightCyan]}Create SSL for site? [Y/n] ${idsCL[Default]}"
		read CREATE_SSL
		showdivide=yes
		if [[ $CREATE_SSL =~ ^[Yy]$ ]] || [ "${CREATE_SSL}" = "" ]; then
			CREATE_SSL=yes
		else
			CREATE_SSL=no
		fi
	fi
	if [ -z ${SITE_TYPE+x} ]; then
		echo -e -n "${idsCL[LightCyan]}Site type ([local]/proxy): ${idsCL[Default]}"
		read SITE_TYPE
		showdivide=yes
		if [ "${SITE_TYPE}" != "proxy" ]; then
			SITE_TYPE=local
		fi
	fi
	if [ "${SITE_TYPE}" = "proxy" ]; then
		if [ -z ${PROXYHOST+x} ]; then
			echo -e -n "${idsCL[LightCyan]}What is the proxy backend address (IP or FQDN): ${idsCL[Default]}"
			read PROXYHOST
			showdivide=yes
		fi
		if [ -z ${PROXYPORT+x} ]; then
			echo -e -n "${idsCL[LightCyan]}What is the proxy backend port (tcp port): ${idsCL[Default]}"
			read PROXYPORT
			showdivide=yes
		fi
		if [ -z ${PROXYSCHEME+x} ]; then
			echo -e -n "${idsCL[LightCyan]}What is the proxy backend scheme (http/https): ${idsCL[Default]}"
			read PROXYSCHEME
			showdivide=yes
		fi
	fi
	if [ "${showdivide}" = "yes" ]; then DIVIDER; fi
	echo ""
	width=14
	printf "%-${width}s: %s\n" "New site" "${NEW_SITE}"
	printf "%-${width}s: %s\n" "Create SSL" "${CREATE_SSL}"
	printf "%-${width}s: %s\n" "Site type" "${SITE_TYPE}"
	if [ "${SITE_TYPE}" = "proxy" ]; then
		printf "%-${width}s: %s\n" "Proxy host" "${PROXYHOST}"
		printf "%-${width}s: %s\n" "Proxy port" "${PROXYPORT}"
		printf "%-${width}s: %s\n" "Proxy scheme" "${PROXYSCHEME}"
	fi
	echo -e -n "${idsCL[LightRed]}Is this information correct? [Y/n]${idsCL[Default]} "
	read response
	if [[ $response =~ ^[Yy]$ ]] || [ "${response}" = "" ]; then 
		if [[ ${NEW_SITE} == *","* ]]; then
			IFS=','; NEW_SITES=(${NEW_SITE}); unset IFS
			MAIN_SITE=${NEW_SITES[0]}
			NGINX_SERVERNAME=${NEW_SITE//[,]/ }
		else
			MAIN_SITE=${NEW_SITE}
			NGINX_SERVERNAME=${NEW_SITE}
				
		fi
		if [ "${SITE_TYPE}" = "proxy" ]; then
			if [ ! -z ${PROXYSCHEME+x} ] && [ ! -z ${PROXYHOST+x} ] && [ ! -z ${PROXYPORT+x} ]; then GO=true; fi
			else GO=true
		fi
		if [ "${GO}" = "true" ]; then
			echo -e "${idsCL[LightGreen]}Setting up new site for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}"
			echo ""
			if [ "${SITE_TYPE}" = "local" ]; then
				echo -e "server {
		listen 8080;" > /etc/nginx/sites-available/${MAIN_SITE}
				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "	listen 8443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "
		server_name ${NGINX_SERVERNAME};
	
		set \$base /var/www/${MAIN_SITE};
		root \$base/public_html;

		access_log  /var/log/nginx/${MAIN_SITE}-access.log;
		error_log  /var/log/nginx/${MAIN_SITE}-error.log warn;" >> /etc/nginx/sites-available/${MAIN_SITE}

				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "
		ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem;
		ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem;
		include conf.d/include/ssl-ciphers.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "
		index index.php;

		location / {
			try_files \$uri \$uri/ /index.php?\$query_string;" >> /etc/nginx/sites-available/${MAIN_SITE}
				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "		include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "	}

		location ~ \.php\$ {
			fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
			include conf.d/include/php_fastcgi.conf;
		}

		include conf.d/include/general.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "	include conf.d/include/letsencrypt-acme-challenge.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE}
				
				for nip in "${NODE_HOSTS[@]}"; do
					if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
					else NCMD="ssh root@${nip}"
					fi
					${NCMD} mkdir -p /var/www/${MAIN_SITE}/{public_html,nginx_logs}
				done
				SET-PERMISSIONS ${MAIN_SITE}
				
			else
				
				echo -e "server {
		set \$forward_scheme ${PROXYSCHEME};
		set \$server         \"${PROXYHOST}\";
		set \$port           ${PROXYPORT};

		listen 8080;" > /etc/nginx/sites-available/${MAIN_SITE}
				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "	listen 8443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "
		server_name ${NGINX_SERVERNAME};" >> /etc/nginx/sites-available/${MAIN_SITE}
				if [ "${CREATE_SSL}" = "yes" ]; then 
					echo -e "
		include conf.d/include/letsencrypt-acme-challenge.conf;
		include conf.d/include/ssl-ciphers.conf;
		ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem;
		ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "
		access_log /var/log/nginx/proxy-${MAIN_SITE}.log proxy;

		location / {" >> /etc/nginx/sites-available/${MAIN_SITE}
				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "		include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "		include conf.d/include/proxy.conf;
		}
	}
	" >> /etc/nginx/sites-available/${MAIN_SITE}
			fi
			ln -s /etc/nginx/sites-available/${MAIN_SITE} /etc/nginx/sites-enabled/${MAIN_SITE}
			if [ "${CREATE_SSL}" = "yes" ]; then
				NEWCERT ${NEW_SITE}
			fi
			
			rm -f  ${FOLDER}/new-site.lastrun
			daterun=`date +%Y-%m-%d-%H-%M-%S`
			echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun
			yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun
			yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun
			daterun=`date +%Y-%m-%d-%H-%M-%S`
			echo -e "${daterun}" >> /etc/nginx/new-site.lastrun
			DIVIDER true
			echo -en "${idsCL[LightYellow]}Waiting for certificate replication between the nodes... ${idsCL[Default]}" 
			checked=false
			timeout=`date --date='2 minutes' +%s`
			until [ "${checked}" = "" ]; do
				tchecked=''
				for nip in "${NODE_HOSTS[@]}"; do
					if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
						else NCMD="ssh root@${nip}"
					fi
					if [ "${NCMD}" != "" ]; then
						tchecked+=`${NCMD} "cat /etc/nginx/new-site.lastrun" | diff - /etc/nginx/new-site.lastrun`
						tchecked+=`${NCMD} "cat /var/www/new-site.lastrun" | diff - /var/www/new-site.lastrun`
					fi
				done
				checked=${tchecked}
				if [ "`date +%s`" -gt "$timeout" ]; then
					echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
					timeout=true
					if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
						echo "Timeout occurred in waiting for replication between nodes." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
					fi
					break
				fi
			done
			if [ "${timeout}" != "true" ]; then
				echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
			fi
			echo
			
			SERVICE nginx reload
			echo ""
			echo -e "${idsCL[LightGreen]}The new site for '${idsCL[LightGreen]}${NEW_SITE}${idsCL[Default]}' has been created.${idsCL[Default]}"
			echo ""
		else
			echo "Missing proxy arguments"
			Exit 1
		fi

	else
		${SCRIPT} newsite
		exit 0
	fi
}

SET-PERMISSIONS(){
	if [ "${1}" != "" ]; then
		NEW_SITE=${1}
		echo -e "${idsCL[Yellow]}Setting new site folder permissions for (/var/www/${NEW_SITE})${idsCL[Default]}"
	fi
	nid=1
	for nip in "${NODE_HOSTS[@]}"
	do
		if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
			nip='localhost '
			NCMD=''
		else
			NCMD="ssh root@${nip}"
		fi
		if [ "${NEW_SITE}" != "" ]; then
			echo -en "${idsCL[Cyan]}iDS-Node${nid} (${nip})    ${idsCL[Default]}"
			${NCMD} chown -R www-data:www-data /var/www/$NEW_SITE
			${NCMD} chmod -R 7775 /var/www/$NEW_SITE
			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
		else
			echo -e "${idsCL[Yellow]}Setting folder permissions for iDS-Node${nid} (${nip})${idsCL[Default]}"
			echo -en "${idsCL[Cyan]}LetsEncrypt Certs    ${idsCL[Default]}"
			${NCMD} chown -R root:letsencrypt /etc/letsencrypt
			${NCMD} chmod -R 6775 /etc/letsencrypt
			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
			echo -en "${idsCL[Cyan]}Webserver folders    ${idsCL[Default]}"
			${NCMD} chown -R www-data:www-data /var/www
			${NCMD} chmod -R 7775 /var/www
			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
        	echo -en "${idsCL[Cyan]}Gitea Folder         ${idsCL[Default]}"
			${NCMD} chown -R git:git /var/lib/gitea
			${NCMD} chmod -R 750 /var/lib/gitea
			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
			echo ""
		fi
		nid=`expr $nid + 1`
	done
	if [ "${NEW_SITE}" != "" ]; then echo ""; fi
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		ENTER2CONTINUE
	fi
}

SERVICES(){
	while :
	do
		echo ""
		echo -e "${idsCL[Green]}Node Services${idsCL[Default]}"
		DIVIDER true
		s=1
		for srvc in "${NODE_SERVICES[@]}"
		do
			echo " [${s}] ${NM_SERVICES[${srvc}]}"
			s=`expr $s + 1`
		done
		echo ""
		if [ -z $action ] || [ "${action}" = "gui" ]; then
			echo " [B] Back"
		fi
		echo " [Q] Quit"
		echo ""
		echo -e -n "${idsCL[LightYellow]}Please select a service from above:${idsCL[Default]} "
		read -n 1 selsrvc
		selsrvc=`expr $selsrvc - 1`
		echo ""
		if [ -z ${NODE_SERVICES[${selsrvc}]} ] && [ "${selsrvc}" != "Q" ] && [ "${selsrvc}" != "q" ] && [ "${selsrvc}" != "B" ] && [ "${selsrvc}" != "b" ]; then
			echo "Thats an invaild option,"
			echo "please select a valid option only."
			sleep 1
			SERVICES
			exit 0
		elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
			exit 0
		elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
			GUI
		else
			echo ""
			echo -e "${idsCL[Green]}Node Service Selected: '${idsST[Bold]}${NM_SERVICES[${NODE_SERVICES[${selsrvc}]}]}${idsST[Reset]}${idsCL[Green]}'${idsCL[Default]}"
			DIVIDER true
			s=1
			for srvcopt in "${NM_SRVCOPT[@]}"; do
				echo " [${s}] ${srvcopt~}"
				s=`expr $s + 1`
			done
			echo ""
			if [ -z $action ] || [ "${action}" = "gui" ]; then
				echo " [B] Back"
			fi
			echo " [Q] Quit"
			echo ""
			echo -e -n "${idsCL[LightYellow]}Please select an action to run from above:${idsCL[Default]} "
			read -n 1 selsrvcopt
			selsrvcopt=`expr $selsrvcopt - 1`
			echo ""
			if [ -z ${NM_SRVCOPT[${selsrvcopt}]} ] && [ "${selsrvcopt}" != "Q" ] && [ "${selsrvcopt}" != "q" ] && [ "${selsrvcopt}" != "B" ] && [ "${selsrvcopt}" != "b" ]; then
				echo "Thats an invaild option,"
				echo "please select a valid option only."
				sleep 1
				SERVICES
				exit 0
			elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
				exit 0
			elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
				SERVICES
				exit 0
			else
				echo
				SERVICE ${NODE_SERVICES[${selsrvc}]} ${NM_SRVCOPT[${selsrvcopt}]}
				echo ""
				if [ -z $action ] || [ "${action}" = "gui" ]; then
					DIVIDER
					ENTER2CONTINUE
				fi
				break
			fi
		fi
	done
	
}

SERVICE(){
	if [ "${NM_SERVICES[${1}]}" = "" ]; then
		echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}"
		exit 1	
	fi
	if [[ "start,stop,restart,reload,enable,disable" != *"${2}"* ]] && [ "${1}" != "daemon-reload" ]; then
		echo -e "${idsCL[Red]}(${3}) is not an allowed service action.${idsCL[Default]}"
		exit 1
	fi
	if [ "$2" = "start" ] &&  [ "$1" = "haproxy" ]; then TACT='restart'; else TACT="${2}"; fi
		
	echo -e "${idsCL[LightGreen]}[[${NM_SERVICES[${1}]} ${NM_SRVCOPTS[${2}]}ing]]${idsCL[Default]}"
	echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
	
	nid=1
	for nip in "${NODE_HOSTS[@]}"
	do
		if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
			nip='localhost '
			NCMD=''
		else
			NCMD="ssh root@${nip}"
		fi
		echo -en "${NM_SRVCOPTS[${2}]}ing on iDS-Node${nid} ($nip)... ${idsCL[Default]}"
		$NCMD systemctl $2 $1
		if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
			echo -e "${idsCL[Green]}OK${idsCL[Default]}"
		elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
			echo -e "${idsCL[Green]}OK${idsCL[Default]}"
		elif [ "$2" = "stop" ]; then
			echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
		else
			echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
		fi
		nid=`expr $nid + 1`
	done
	
	echo ""
	echo -e "${idsCL[Green]}${NM_SERVICES[${1}]} has been ${NM_SRVCOPTS[${2}]}ed${idsCL[Default]}"
	echo ""
}

GUI(){
	DISP_HEADER true true
	while :
	do
		echo " [1] Node Status"
		echo " [2] Node Services"
		echo ""
		echo " [3] New Site"
		echo " [4] Delete Site"
		echo ""
		echo " [5] New Certificate"
		echo " [6] Renew Certificates"
		echo " [7] List Certificates"
		echo ""
		echo " [8] Set folder permissions on nodes"
		echo " [9] Backup Node Settings and Files"
		echo ""
		echo " [Q] Quit"
		echo ""
		echo ""
		echo -e -n "${idsCL[LightYellow]}Please select an [ActionItem] from above:${idsCL[Default]} "
		read -n 1 opt
		echo ""
		case $opt in
			1) STATUS; GUI;;
			2) SERVICES; GUI;;
			3) NEWSITE; GUI;;
			4) DELSITES; GUI;;
			5) NEWCERT; GUI;;
			6) CERTRENEW; GUI;;
			7) LISTCERTS; GUI;;
			8) SET-PERMISSIONS; GUI;;
			9) BACKUP; GUI;;
			[Qq]) EXIT1
				exit 0;;
			*)	echo "Thats an invaild option,";
				echo "please select a valid option only.";
				sleep 1;;
		esac
	done
}
if [ ${action-x} ]; then
	if [[ "${noheader}" != *" ${action} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
		DISP_HEADER
	fi
	case $action in
		newcert) NEWCERT ${2};;
		certrenew) CERTRENEW;;
		listcerts) LISTCERTS;;
		checkcerts) CHECK-CERTS;;
		nightlyrenew) NIGHTLYRENEW;;
		delsite) DELSITE ${2} ${3} ${4} ${5};;
		delsites) DELSITES;;
		newsite) NEWSITE ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14};;
		update) ;;
		backup) BACKUP;;
		status) STATUS ${2};;
		status-check) STATUS-CHECK ${2};;
		set-permissions) DIVIDER; SET-PERMISSIONS ${2};;
		service) SERVICE ${2} ${3};;
		services) SERVICES;;
		gui) GUI;;
		*)
			echo -e "Usage: ${idsCL[Yellow]}nodemgmt${idsCL[Default]} { ${idsCL[Yellow]}service${idsCL[Default]} [service] [action] | ${idsCL[Yellow]}status${idsCL[Default]} | ${idsCL[Yellow]}newcert${idsCL[Default]} [domain(,s)] | ${idsCL[Yellow]}certrenew${idsCL[Default]} }"
			echo ""
			echo ""
			exit 0;;
	esac
else
	GUI
fi

exit 0