From 5d18be16e606f01663bdf4a5cca8031fa3c4a90d Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Sun, 10 Feb 2019 16:57:27 -0600
Subject: [PATCH] revert

---
 nodemgmt-scripts copy.sh | 1279 ++++++++++++++++++++++++++++++++++++++
 nodemgmt-scripts.sh      |   17 -
 2 files changed, 1279 insertions(+), 17 deletions(-)
 create mode 100755 nodemgmt-scripts copy.sh

diff --git a/nodemgmt-scripts copy.sh b/nodemgmt-scripts copy.sh
new file mode 100755
index 00000000..8e725b41
--- /dev/null
+++ b/nodemgmt-scripts copy.sh	
@@ -0,0 +1,1279 @@
+#!/usr/bin/env bash
+# NodeMgmt - Galera/NGINX Node Management Scripts
+
+action="$1"
+
+if [ ! -f /opt/idssys/settings/nodemgmt.conf ]; then
+	if [ ! -d /opt/idssys/settings ]; then
+		mkdir /opt/idssys/settings
+	fi
+	mv /opt/idssys/nodemgmt/settings.conf /opt/idssys/settings/nodemgmt.conf
+fi
+
+source /opt/idssys/defaults/colors.inc
+source /opt/idssys/defaults/default.inc
+source /opt/idssys/settings/nodemgmt.conf
+source /opt/idssys/nodemgmt/defaults.inc
+
+# ========================================================= #
+
+STATUS(){
+	start=`date +%s`
+	ST_ACTION=${1}
+	if [ "${ST_ACTION}" = "report" ]; then
+		unset idsCL idsBG idsST
+		idsCL=('')
+		idsBG=('')
+		idsST=('')
+	fi
+	declare -i cw; declare -i spc1; declare -i c
+	########################
+	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ]; then
+		if [ "${ST_ACTION}" != "check" ]; then
+			echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}"
+			echo
+		fi
+		if [ ! -z ${LOCAL_SERVICES+x} ];
+			then NHCMD='ssh root@10.5.10.51'
+			else NHCMD=''
+		fi
+		for rcheck in "${REPL_CHECK[@]}"; do
+			${NHCMD} rm -f ${FOLDER}/test.repl
+			daterun=`date +%Y-%m-%d-%H-%M-%S`
+			if [ "${NHCMD}" = "" ]; then
+				echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
+				yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
+			else
+				${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
+				yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
+			fi
+		done
+		if [ "${ST_ACTION}" = "repl" ]; then
+			sleep 5
+		fi
+	fi
+	########################
+	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then
+		if [ "${ST_ACTION}" != "check" ]; then
+			echo -e "${idsST[Bold]}"; DIVIDER
+			echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}"
+			DIVIDER; echo -e "${idsST[Reset]}"
+		fi
+		if [ ! -z ${LOCAL_SERVICES+x} ]; then
+			if [ "${ST_ACTION}" != "check" ]; then
+				lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
+				uptime=`uptime -p`
+				echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
+				DIVIDER false green
+			fi
+			for srvc in "${LOCAL_SERVICES[@]}"; do
+				if [ "${ST_ACTION}" != "check" ]; then
+					c=0; cw=22; spc=''
+					spc1=${cw}-${#NM_SERVICES[${srvc}]}
+					until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+					echo -en " ${NM_SERVICES[${srvc}]}$spc: "
+				fi
+				
+				if [ "$(systemctl is-active ${srvc})" != "active" ]; then
+					if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
+						if [ "${ST_ACTION}" != "check" ]; then
+							echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
+						fi
+						echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}
+						touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
+					else
+						errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
+						if [ $errtime -gt ${RENOTIFY} ]; then
+							if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
+								mv ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
+							fi
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+							fi
+							echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}
+							touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
+						fi
+					fi
+				else
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+					fi
+					if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
+						if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
+						else
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
+						fi
+						echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}
+						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
+						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
+					fi
+				fi
+			done
+			
+			if [ "${ST_ACTION}" != "check" ]; then echo; fi
+		fi
+		########################
+		nid=1
+		for nip in "${NODE_HOSTS[@]}"; do
+			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
+				else NCMD="ssh root@${nip}"; LH=''
+			fi
+			if [ "${ST_ACTION}" != "check" ]; then
+				uptime=`${NCMD} uptime -p`
+				echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+				DIVIDER false green
+			fi
+			if [ "${NCMD}" != "" ]; then
+				checkhost=$(CHECK_HOST ${nip})
+			fi
+			if [ "${checkhost}" != "false" ]; then
+				if [ -f ${FOLDER}/${nip}.down ]; then
+					if [ -f ${FOLDER}/${nip}.errtime ]; then
+						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+					else
+						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+					fi
+					rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
+					echo "iDS-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
+				fi
+				for srvc in "${NODE_SERVICES[@]}"; do
+					if [ "${ST_ACTION}" != "check" ]; then
+						c=0; cw=22; spc=''
+						spc1=${cw}-${#NM_SERVICES[${srvc}]}
+						until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+						echo -en " ${NM_SERVICES[${srvc}]}$spc: "
+					fi
+					
+					if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
+						if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
+							fi
+							echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+							touch ${FOLDER}/${nip}-${srvc}.down
+						else
+							errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+							if [ $errtime -gt ${RENOTIFY} ]; then
+								if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+									mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
+								fi
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+								if [ "${ST_ACTION}" != "check" ]; then
+									echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+								fi
+								echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+								touch ${FOLDER}/${nip}-${srvc}.down
+							fi
+						fi
+					else
+						if [ "${ST_ACTION}" != "check" ]; then
+							echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+						fi
+						if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
+							if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+							else
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+							fi
+							echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+							rm -f ${FOLDER}/${nip}-${srvc}.down
+							rm -f ${FOLDER}/${nip}-${srvc}.errtime
+						fi
+					fi
+				done
+			else
+				errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+				if [ $errtime -gt ${RENOTIFY} ]; then
+					if [ ! -f ${FOLDER}/${nip}.errtime ]; then
+						mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
+					fi
+					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+					fi
+					echo "iDS-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+				else
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
+					fi
+				fi
+				touch ${FOLDER}/${nip}.down
+			fi
+			if [ "${ST_ACTION}" != "check" ]; then echo; fi
+			nid=`expr $nid + 1`
+		done
+	fi
+	########################
+	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then
+		if [ "${ST_ACTION}" != "check" ]; then
+			echo -e "${idsST[Bold]}"; DIVIDER
+			echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
+			DIVIDER; echo -e "${idsST[Reset]}"
+		fi
+		nid=1
+		for nip in "${NODE_HOSTS[@]}"; do
+			if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
+			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
+				if [ "${ST_ACTION}" != "check" ]; then
+					echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
+					DIVIDER false green
+				fi
+				checkhost=$(CHECK_HOST ${nip})
+				if [ "${checkhost}" != "false" ]; then
+					if [ -f ${FOLDER}/${nip}.down ]; then
+						rm -f ${FOLDER}/${nip}.down
+						echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
+					fi
+					for rcheck in "${REPL_CHECK[@]}"; do
+						if [ "${ST_ACTION}" != "check" ]; then
+							c=0; cw=22; spc=''
+							spc1=${cw}-${#REPL_DESC[${rcheck}]}
+							timeout=`date --date='20 seconds' +%s`
+							until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+							echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
+						fi
+						checked=false
+						until [ "${checked}" = "" ]; do
+							if [ "${NHCMD}" = "" ]; then
+								checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
+							else
+								checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
+							fi
+							if [ "`date +%s`" -gt "$timeout" ]; then
+								timeout=true
+								break
+							fi
+						done
+						if [ "${timeout}" = "true" ]; then
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
+							fi
+							if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+								touch ${FOLDER}/${nip}-${rcheck}.down
+								echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
+							fi
+						else
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Green]}Good${idsCL[Default]}"
+							fi
+							if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+								rm -f ${FOLDER}/${nip}-${rcheck}.down
+								echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
+							fi
+						fi
+						
+					done
+				else
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
+					fi
+					if [ ! -f ${FOLDER}/${nip}.down ]; then
+						touch ${FOLDER}/${nip}.down
+						echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
+					fi
+				fi
+				if [ "${ST_ACTION}" != "check" ]; then echo; fi
+			fi
+			nid=`expr $nid + 1`
+		done
+	fi
+	
+	if [ "${ST_ACTION}" != "check" ]; then
+		echo ""
+		if [ -z $action ] || [ "${action}" = "gui" ]; then
+			DIVIDER true
+			ENTER2CONTINUE
+		fi
+	fi
+	end=`date +%s`
+	runtime=$((end-start))
+	echo "runtime: ${runtime}"
+}
+
+# STATUS-CHECK(){
+# 	start=`date +%s`
+# 	if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
+# 		if [ ! -z ${LOCAL_SERVICES+x} ]; then NHCMD='ssh root@10.5.10.51'
+# 		else NHCMD=''; fi
+# 		# for rcheck in "${REPL_CHECK[@]}"; do
+# 		# 	${NHCMD} rm -f ${FOLDER}/test.repl
+# 		# 	daterun=`date +%Y-%m-%d-%H-%M-%S`
+# 		# 	if [ "${NHCMD}" = "" ]; then
+# 		# 		echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
+# 		# 		yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
+# 		# 	else
+# 		# 		${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
+# 		# 		yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
+# 		# 	fi
+# 		# done
+# 		if [ ! -z ${LOCAL_SERVICES+x} ]; then
+# 			lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
+# 			for srvc in "${LOCAL_SERVICES[@]}"; do
+# 				if [ "$(systemctl is-active ${srvc})" != "active" ]; then
+# 					if [ ! -f ${FOLDER}/localhost-${srvc}.down ]; then
+# 						echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
+# 						touch ${FOLDER}/localhost-${srvc}.down
+# 					fi
+# 				elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then
+# 					echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
+# 					rm -f ${FOLDER}/localhost-${srvc}.down
+# 					rm -f ${FOLDER}/localhost-${srvc}.errtime
+# 				fi
+# 			done
+# 		fi
+# 		if [ -z ${LOCAL_SERVICES+x} ] || [ "${1}" = "all" ]; then
+# 			nid=1
+# 			for nip in "${NODE_HOSTS[@]}"; do
+# 				checkhost=$(CHECK_HOST ${nip})
+# 				if [ "${checkhost}" != "false" ]; then
+# 					if [ -f ${FOLDER}/${nip}.down ]; then
+# 						rm -f ${FOLDER}/${nip}.down
+# 						echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
+# 					fi
+# 					if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
+# 						else NCMD="ssh root@${nip}"
+# 					fi
+# 					for srvc in "${NODE_SERVICES[@]}"; do
+# 						if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
+# 							if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
+# 								echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+# 								touch ${FOLDER}/${nip}-${srvc}.down
+# 							else
+# 								errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+# 								if [ $errtime -gt ${RENOTIFY} ]; then
+# 									if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+# 										mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
+# 									fi
+# 									toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+# 									echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+# 									touch ${FOLDER}/${nip}-${srvc}.down
+# 								fi
+# 							fi
+# 						elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
+# 							echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+# 							rm -f ${FOLDER}/${nip}-${srvc}.down
+# 							rm -f ${FOLDER}/${nip}-${srvc}.errtime
+# 						fi
+# 					done
+# 				elif [ ! -f ${FOLDER}/${nip}.down ]; then
+# 					touch ${FOLDER}/${nip}.down
+# 					echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
+# 				fi
+# 				nid=`expr $nid + 1`
+# 			done
+# 			##REPLICATION CHECK
+# 			# nid=1
+# 			# for nip in "${NODE_HOSTS[@]}"; do
+# 			# 	if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
+# 			# 	if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
+# 			# 		checkhost=$(CHECK_HOST ${nip})
+# 			# 		if [ "${checkhost}" != "false" ]; then
+# 			# 			if [ -f ${FOLDER}/${nip}.down ]; then
+# 			# 				rm -f ${FOLDER}/${nip}.down
+# 			# 				echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
+# 			# 			fi
+# 			# 			for rcheck in "${REPL_CHECK[@]}"; do
+# 			# 				timeout=`date --date='20 seconds' +%s`
+# 			# 				checked=false
+# 			# 				until [ "${checked}" = "" ]; do
+# 			# 					if [ "${NHCMD}" = "" ]; then
+# 			# 						checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
+# 			# 					else
+# 			# 						checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
+# 			# 					fi
+# 			# 					if [ "`date +%s`" -gt "$timeout" ]; then
+# 			# 						timeout=true
+# 			# 						break
+# 			# 					fi
+# 			# 				done
+# 			# 				if [ "${timeout}" != "true" ]; then
+# 			# 					if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+# 			# 						rm -f ${FOLDER}/${nip}-${rcheck}.down
+# 			# 						echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
+# 			# 					fi
+# 			# 				elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+# 			# 					touch ${FOLDER}/${nip}-${rcheck}.down
+# 			# 					echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
+# 			# 				fi
+# 			# 			done
+# 			# 		elif [ ! -f ${FOLDER}/${nip}.down ]; then
+# 			# 			touch ${FOLDER}/${nip}.down
+# 			# 			echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
+# 			# 		fi
+# 			# 	fi
+# 			# 	nid=`expr $nid + 1`
+# 			# done
+# 		fi
+# 	fi
+# 	end=`date +%s`
+# 	runtime=$((end-start))
+# 	echo "runtime: ${runtime}"
+# }
+
+NEWCERT(){
+	echo ""
+	if [ -z ${1+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}Create certificate for what address: ${idsCL[Default]}"
+		read NEW_CERT
+		echo ""
+	else
+		NEW_CERT=${1}
+	fi
+	if [[ ${NEW_CERT} == *","* ]]; then
+		IFS=','; NEW_CERTS=(${NEW_CERT}); unset IFS
+		MAIN_CERT=${NEW_CERTS[0]}
+	else
+		MAIN_CERT=${NEW_CERT}
+	fi
+	if [ "${NEW_CERT}" != "" ]; then
+		echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}"
+		echo ""
+		
+		for nip in "${NODE_HOSTS[@]}"; do
+			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then 
+				`ssh root@${nip} service nginx stop`
+			fi
+		done
+		
+		# $CERT_DAEMON certonly --webroot -w /var/www/html -d ${NEW_CERT}
+		$CERT_DAEMON certonly --dry-run --webroot -w /var/www/html -d ${NEW_CERT}
+		if [ -f /etc/letsencrypt/live/${MAIN_CERT}/cert.pem ]; then
+			chown -R root:letsencrypt /etc/letsencrypt
+			chmod -R 6775 /etc/letsencrypt
+			if [ -f /etc/nginx/sites-available/${MAIN_CERT} ]; then
+				rm -f  ${FOLDER}/cert-request.lastrun
+				daterun=`date +%Y-%m-%d-%H-%M-%S`
+				echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun
+				yes | cp -rfH ${FOLDER}/cert-request.lastrun /etc/letsencrypt/cert-request.lastrun
+				DIVIDER true
+				echo -e "Since this certificate is tied to a site in NGINX, we will reload NGINX across the nodes."
+				echo
+				echo -en "${idsCL[LightYellow]}Waiting for certificate replication across the nodes... ${idsCL[Default]}"
+				checked=false
+				timeout=`date --date='2 minutes' +%s`
+				until [ "${checked}" = "" ]; do
+					tchecked=''
+					for nip in "${NODE_HOSTS[@]}"; do
+						if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
+							else NCMD="ssh root@${nip}"
+						fi
+						if [ "${NCMD}" != "" ]; then
+							tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-request.lastrun" | diff - /etc/letsencrypt/cert-request.lastrun`
+						fi
+					done
+					checked=${tchecked}
+					if [ "`date +%s`" -gt "$timeout" ]; then
+						echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
+						timeout=true
+						echo "Timeout occurred in waiting for replication between nodes. (${NEW_CERT})" | mail -s "Cert-Request" ${STATUS_CHECK_EMAIL}
+						break
+					fi
+				done
+				if [ "${timeout}" != "true" ]; then
+					echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
+				fi
+				echo
+				SERVICE nginx reload
+				for nip in "${NODE_HOSTS[@]}"; do
+					if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then 
+						`ssh root@${nip} service nginx start`
+					fi
+				done
+			fi
+			echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}"
+		else
+			echo ""
+			echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}"
+		fi
+		echo ""
+		if [ -z $action ] || [ "${action}" = "gui" ]; then
+			DIVIDER true
+			ENTER2CONTINUE
+		fi
+	else
+		echo "You havent entered a site address."
+		sleep 1
+		NEWCERT
+		exit 0
+	fi
+}
+
+CERTRENEW(){
+	echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}"
+	echo
+	$CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
+	chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
+	chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
+	yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
+	daterun=`date +%Y-%m-%d-%H-%M-%S`
+	echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
+	DIVIDER true
+	echo -en "${idsCL[LightYellow]}Waiting for certificate replication between the nodes... ${idsCL[Default]}" 
+	checked=false
+	timeout=`date --date='2 minutes' +%s`
+	until [ "${checked}" = "" ]; do
+		tchecked=''
+		for nip in "${NODE_HOSTS[@]}"; do
+			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
+				else NCMD="ssh root@${nip}"
+			fi
+			if [ "${NCMD}" != "" ]; then
+				tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-renewal.lastrun" | diff - /etc/letsencrypt/cert-renewal.lastrun`
+			fi
+		done
+		checked=${tchecked}
+		if [ "`date +%s`" -gt "$timeout" ]; then
+			echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
+			timeout=true
+			if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
+				echo "Timeout occurred in waiting for replication between nodes." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
+			fi
+			break
+		fi
+	done
+	if [ "${timeout}" != "true" ]; then
+		echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
+	fi
+	echo
+	SERVICE nginx reload 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
+	echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}"
+	echo ""
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		DIVIDER true
+		ENTER2CONTINUE
+	fi
+}
+NIGHTLYRENEW(){
+	rm -f ${FOLDER}/cert-renewal.lastrun
+	$CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun
+	chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
+	chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
+	yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
+	daterun=`date +%Y-%m-%d-%H-%M-%S`
+	echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
+	checked=false
+	timeout=`date --date='2 minutes' +%s`
+	until [ "${checked}" = "" ]; do
+		tchecked=''
+		for nip in "${NODE_HOSTS[@]}"; do
+			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
+				else NCMD="ssh root@${nip}"
+			fi
+			if [ "${NCMD}" != "" ]; then
+				tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-renewal.lastrun" | diff - /etc/letsencrypt/cert-renewal.lastrun`
+			fi
+		done
+		checked=${tchecked}
+		if [ "`date +%s`" -gt "$timeout" ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then
+			echo "Timeout occurred in waiting for replication between nodes." | mail -s "Nightly-Cert-Renewal" ${STATUS_CHECK_EMAIL}
+			break
+		fi
+	done
+	SERVICE nginx reload &>> ${FOLDER}/cert-renewal.lastrun
+}
+
+LISTCERTS(){
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		DIVIDER true
+	fi
+	echo 
+	echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}"
+	DIVIDER
+	echo -e "Subject Name                  Expires       Alternate Subjects"
+	DIVIDER
+	for certdir in /etc/letsencrypt/live/*/ ; do
+		echo $certdir
+		SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
+		SUBJECTNAMES=$(openssl x509 -in ${certdir}/cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV)
+		SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, }
+		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g")
+		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g")
+		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g")
+		CERTEXPIRE=$(date -d "$(: | openssl x509 -in ${certdir}/cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s'); 
+		DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS)));
+		
+		AWK=$(which awk)
+		DATE=$(which date)
+		GREP=$(which grep)
+		OPENSSL=$(which openssl)
+		SED=$(which sed)
+
+		
+        #CERTDATE=$(${OPENSSL} x509 -in ${certdir}/cert.pem -text -enddate | grep 'Not After')
+		
+        # COMMONNAME=$(${OPENSSL} x509 -in ${certdir}/cert.pem -text -subject | \
+        #            ${SED} -e 's/.*CN=//' | \
+        #            ${SED} -e 's/\/.*//')
+					   
+		#SUBJECT=${COMMONNAME}
+
+
+		declare -i cw; declare -i spc1; declare -i c
+		c=0; cw=30; spc=''
+		spc1=${cw}-${#SUBJECT}
+		until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+		if [ $DUEIN -gt $CERTEXPIRE ]; then
+			date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}"
+		else
+			date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}"
+		fi
+		echo -e "${idsCL[Cyan]}${SUBJECT}${spc}${date}    ${idsCL[Cyan]}${SUBJECTNAMES}${idsCL[Default]}" 
+	done
+	echo 
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		DIVIDER true
+		ENTER2CONTINUE
+	fi
+	echo ""
+}
+BACKUP(){
+	BACKUP_FOLDER=/opt/idssys/backups/node-backup
+	#BACKUP_FOLDER=/opt/idssys/backups/node-backups/${NODE_HOSTNAME} #/`date +%Y-%m-%d`
+	#BACKUP_TIMEFOLDER=${BACKUP_DAYFOLDER}/`date +%H-%M`
+	if [ "${action}" != "backup" ]; then
+		echo -e "${idsCL[Green]}Backing up Node Settings and Files...${idsCL[Default]}"
+		DIVIDER true
+	fi
+	if [ ! -d ${BACKUP_FOLDER} ]; then
+		mkdir -p ${BACKUP_FOLDER}
+	fi
+	declare -A BACKUP_ITEMS
+	BACKUP_ITEMS[nginx-settings]=/etc/nginx
+	BACKUP_ITEMS[letsencrypt-certs]=/etc/letsencrypt
+	BACKUP_ITEMS[gitea]=/var/lib/gitea
+	BACKUP_ITEMS[nginx-logs]='/var/www/!NGINX-Logs'
+	#BACKUP_ITEMS[webserver-files]=/var/www
+	for item in "${!BACKUP_ITEMS[@]}"
+	do
+		if [ "${action}" != "backup" ]; then
+			echo -en "Backing up '${item}'... "
+		fi
+		#tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${BACKUP_ITEMS[$item]} .
+		tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -g ${BACKUP_FOLDER}/backup.snar  -C ${BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git'
+		if [ "${action}" != "backup" ]; then
+			echo -e "${idsCL[Green]}Ok${idsCL[Default]}"
+		fi
+	done
+	touch ${BACKUP_FOLDER}/backup.lastrun
+	if [ "${action}" != "backup" ]; then
+		echo ""
+		DIVIDER true
+	fi
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		ENTER2CONTINUE
+	fi
+}
+
+DELSITE(){
+	while [ $# -gt 0 ]; do
+	    case "$1" in
+	        -site) DEL_SITE=${2};;
+	        -ssl) DEL_SSL=${2};;
+			-list) DELSITES; exit 0;;
+	   	 	-*)
+				echo "Invalid option: '${1}' requires an argument" 1>&2
+				echo ""
+				echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
+				width=33
+				printf "%-${width}s- %s\n" "    -site {FQDN address}" "(*required)"
+				printf "%-${width}s- %s\n" "    -ssl {yes or [no]}" "Delete SSL certs as well"
+				printf "%-${width}s- %s\n" "    -list" "List sites (same as running nodemgmt delsites)"
+				echo "}"
+				exit 1;;
+	    esac
+		shift
+	done
+	if [ -z ${DEL_SITE+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}Delete what site address: ${idsCL[Default]}"
+		read DEL_SITE
+		echo ""
+	fi
+	if [[ $DEL_SSL =~ ^[Nn]$ ]]; then
+		DEL_SSL=no
+	elif [[ $DEL_SSL =~ ^[Yy]$ ]]; then
+		DEL_SSL=yes
+	elif [ -z ${DEL_SSL+x} ]; then
+		echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${DEL_SITE}' as well? [y/N]${idsCL[Default]} "
+		read DEL_SSL
+	fi
+	if [ ! -z ${DEL_SITE+x} ] && [ "${DEL_SITE}" != "" ]; then
+		echo -e "${idsCL[LightRed]}Deleting site '${idsCL[Red]}${DEL_SITE}${idsCL[LightRed]}'...${idsCL[Default]}"
+		echo ""
+	
+		echo -e "${idsCL[LightRed]}[[Removing Files and Folders]]${idsCL[Default]}"
+		echo -e "${idsCL[LightRed]}-------------------------------------------${idsCL[Default]}"
+
+		nid=1
+		for nip in "${NODE_HOSTS[@]}"
+		do
+			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
+				nip='localhost '
+				NCMD=''
+			else
+				NCMD="ssh root@${nip}"
+			fi
+			echo -en "Removing from iDS-Node${nid} ($nip)... ${idsCL[Default]}"
+			#if [ -f /etc/nginx/sites-available/${DEL_SITE} ]; then
+				${NCMD} rm -f /etc/nginx/sites-available/${DEL_SITE}
+				${NCMD} rm -f /etc/nginx/sites-enabled/${DEL_SITE}
+				#fi
+			#if [ -d /var/www/${DEL_SITE} ]; then
+				${NCMD} rm -rf /var/www/${DEL_SITE}
+				#fi
+			if [ "${DEL_SSL}" = "yes" ]; then
+				${NCMD} rm -rf /etc/letsencrypt/archive/${DEL_SITE}
+				${NCMD} rm -rf /etc/letsencrypt/live/${DEL_SITE}
+				${NCMD} rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf
+			fi
+			echo -e "${idsCL[Green]}OK${idsCL[Default]}"
+			nid=`expr $nid + 1`
+		done
+		echo ""
+		SERVICE nginx reload
+		echo -e "${idsCL[LightRed]}Site has been deleted.${idsCL[Default]}"
+	else
+		echo "Missing arguments"
+		echo ""
+		echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
+		width=33
+		printf "%-${width}s- %s\n" "    -site {FQDN address}" "Site to delete"
+		printf "%-${width}s- %s\n" "    -ssl {yes or [no]}" "Delete SSL certs as well"
+		printf "%-${width}s- %s\n" "    -list" "List sites (same as running nodemgmt delsites)"
+		echo "}"
+		exit 1
+	fi
+}
+
+DELSITES(){
+	echo
+	echo -e "${idsCL[Red]}Select a site to delete...${idsCL[Default]}"
+	DIVIDER true
+	sid=1
+	for siteconf in /etc/nginx/sites-available/* ; do
+		[ -e "$siteconf" ] || continue
+		IFS='/'; site_conf=(${siteconf}); unset IFS
+		SITES[${sid}]=${site_conf[4]}
+		sid=`expr $sid + 1`
+	done
+	for s in "${!SITES[@]}"; do
+		echo " [${s}] ${SITES[${s}]}"
+	done
+	echo ""
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		echo " [B] Back"
+	fi
+	echo " [Q] Quit"
+	echo ""
+	echo -e -n "${idsCL[LightYellow]}Please select a site from above from above:${idsCL[Default]} "
+	read selsite
+	echo ""
+	if [ -z ${SITES[$selsite]} ] && [ "${selsite}" != "Q" ] && [ "${selsite}" != "q" ] && [ "${selsite}" != "B" ] && [ "${selsite}" != "b" ]; then
+		echo "Thats an invaild option,"
+		echo "please select a valid option only."
+		sleep 1
+		DELSITES
+		exit 0
+	elif [ "${selsite}" = "Q" ] || [ "${selsite}" = "q" ]; then
+		exit 0
+	elif [ "${selsite}" = "B" ] || [ "${selsite}" = "b" ]; then
+		GUI
+	else
+		while :
+		do
+			echo -e -n "${idsCL[LightRed]}Are you sure you want to delete '${idsST[Bold]}${idsCL[Red]}${SITES[${selsite}]}${idsST[Reset]}${idsCL[LightRed]}'? [y/N]${idsCL[Default]} "
+			read response
+			echo ""
+			if [[ $response =~ ^[Yy]$ ]]; then
+				echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${SITES[${selsite}]}' as well? [y/N]${idsCL[Default]} "
+				read sslresponse
+				DELSITE -site ${SITES[${selsite}]} -ssl ${sslresponse}
+				echo ""
+				DIVIDER
+				ENTER2CONTINUE
+				break
+			else
+				break
+			fi			
+		done
+		DELSITES
+		exit 0
+	fi
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		ENTER2CONTINUE
+	fi
+}
+
+NEWSITE(){
+	echo
+	while [ $# -gt 0 ]; do
+		case "$1" in
+			-site) NEW_SITE=${2};;
+			-type) SITE_TYPE=${2};;
+			-ssl) CREATE_SSL=${2};;
+			-proxy_scheme) PROXYSCHEME=${2};;
+			-proxy_host) PROXYHOST=${2};;
+			-proxy_port) PROXYPORT=${2};;
+			-h | -help | --help)
+				echo "Missing arguments"
+				echo ""
+				echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {"
+				width=33
+				printf "%-${width}s- %s\n" "    -site {FQDN address(,es)}" "(new site and aliases, comma separated)"
+				printf "%-${width}s- %s\n" "    -ssl {yes or no}" "(defaults to yes)"
+				printf "%-${width}s- %s\n" "    -type {'local' or 'proxy'}" "(defaults to local)"
+				printf "%-${width}s- %s\n" "    -port {host port}" "(proxy backend host)"
+				printf "%-${width}s- %s\n" "    -host {IP or FQDN}" "(proxy backend port)"
+				printf "%-${width}s- %s\n" "    -scheme {http or https}" "(proxy backend scheme)"
+				echo "}"
+				exit 1;;
+		esac
+		shift
+	done
+		
+	#if [ -z ${SITE_TYPE+x} ]; then SITE_TYPE=local; fi
+	#if [ -z ${CREATE_SSL+x} ]; then CREATE_SSL=true; fi
+	if [ -z ${NEW_SITE+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}New site domain name: ${idsCL[Default]}"
+		read NEW_SITE
+		showdivide=yes
+	fi
+	if [ -z ${CREATE_SSL+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}Create SSL for site? [Y/n] ${idsCL[Default]}"
+		read CREATE_SSL
+		showdivide=yes
+		if [[ $CREATE_SSL =~ ^[Yy]$ ]] || [ "${CREATE_SSL}" = "" ]; then
+			CREATE_SSL=yes
+		else
+			CREATE_SSL=no
+		fi
+	fi
+	if [ -z ${SITE_TYPE+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}Site type ([local]/proxy): ${idsCL[Default]}"
+		read SITE_TYPE
+		showdivide=yes
+		if [ "${SITE_TYPE}" != "proxy" ]; then
+			SITE_TYPE=local
+		fi
+	fi
+	if [ "${SITE_TYPE}" = "proxy" ]; then
+		if [ -z ${PROXYHOST+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}What is the proxy backend address (IP or FQDN): ${idsCL[Default]}"
+			read PROXYHOST
+			showdivide=yes
+		fi
+		if [ -z ${PROXYPORT+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}What is the proxy backend port (tcp port): ${idsCL[Default]}"
+			read PROXYPORT
+			showdivide=yes
+		fi
+		if [ -z ${PROXYSCHEME+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}What is the proxy backend scheme (http/https): ${idsCL[Default]}"
+			read PROXYSCHEME
+			showdivide=yes
+		fi
+	fi
+	if [ "${showdivide}" = "yes" ]; then DIVIDER; fi
+	echo ""
+	width=14
+	printf "%-${width}s: %s\n" "New site" "${NEW_SITE}"
+	printf "%-${width}s: %s\n" "Create SSL" "${CREATE_SSL}"
+	printf "%-${width}s: %s\n" "Site type" "${SITE_TYPE}"
+	if [ "${SITE_TYPE}" = "proxy" ]; then
+		printf "%-${width}s: %s\n" "Proxy host" "${PROXYHOST}"
+		printf "%-${width}s: %s\n" "Proxy port" "${PROXYPORT}"
+		printf "%-${width}s: %s\n" "Proxy scheme" "${PROXYSCHEME}"
+	fi
+	echo -e -n "${idsCL[LightRed]}Is this information correct? [Y/n]${idsCL[Default]} "
+	read response
+	if [[ $response =~ ^[Yy]$ ]] || [ "${response}" = "" ]; then 
+		if [[ ${NEW_SITE} == *","* ]]; then
+			IFS=','; NEW_SITES=(${NEW_SITE}); unset IFS
+			MAIN_SITE=${NEW_SITES[0]}
+			NGINX_SERVERNAME=${NEW_SITE//[,]/ }
+		else
+			MAIN_SITE=${NEW_SITE}
+			NGINX_SERVERNAME=${NEW_SITE}
+				
+		fi
+		if [ "${SITE_TYPE}" = "proxy" ]; then
+			if [ ! -z ${PROXYSCHEME+x} ] && [ ! -z ${PROXYHOST+x} ] && [ ! -z ${PROXYPORT+x} ]; then GO=true; fi
+			else GO=true
+		fi
+		if [ "${GO}" = "true" ]; then
+			echo -e "${idsCL[LightGreen]}Setting up new site for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}"
+			echo ""
+			if [ "${SITE_TYPE}" = "local" ]; then
+				echo -e "server {
+		listen 8080;" > /etc/nginx/sites-available/${MAIN_SITE}
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "	listen 8443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "
+		server_name ${NGINX_SERVERNAME};
+	
+		set \$base /var/www/${MAIN_SITE};
+		root \$base/public_html;
+
+		access_log  /var/log/nginx/${MAIN_SITE}-access.log;
+		error_log  /var/log/nginx/${MAIN_SITE}-error.log warn;" >> /etc/nginx/sites-available/${MAIN_SITE}
+
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "
+		ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem;
+		ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem;
+		include conf.d/include/ssl-ciphers.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "
+		index index.php;
+
+		location / {
+			try_files \$uri \$uri/ /index.php?\$query_string;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "		include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "	}
+
+		location ~ \.php\$ {
+			fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
+			include conf.d/include/php_fastcgi.conf;
+		}
+
+		include conf.d/include/general.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "	include conf.d/include/letsencrypt-acme-challenge.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE}
+				
+				for nip in "${NODE_HOSTS[@]}"; do
+					if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
+					else NCMD="ssh root@${nip}"
+					fi
+					${NCMD} mkdir -p /var/www/${MAIN_SITE}/{public_html,nginx_logs}
+				done
+				SET-PERMISSIONS ${MAIN_SITE}
+				
+			else
+				
+				echo -e "server {
+		set \$forward_scheme ${PROXYSCHEME};
+		set \$server         \"${PROXYHOST}\";
+		set \$port           ${PROXYPORT};
+
+		listen 8080;" > /etc/nginx/sites-available/${MAIN_SITE}
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "	listen 8443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "
+		server_name ${NGINX_SERVERNAME};" >> /etc/nginx/sites-available/${MAIN_SITE}
+				if [ "${CREATE_SSL}" = "yes" ]; then 
+					echo -e "
+		include conf.d/include/letsencrypt-acme-challenge.conf;
+		include conf.d/include/ssl-ciphers.conf;
+		ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem;
+		ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "
+		access_log /var/log/nginx/proxy-${MAIN_SITE}.log proxy;
+
+		location / {" >> /etc/nginx/sites-available/${MAIN_SITE}
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "		include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "		include conf.d/include/proxy.conf;
+		}
+	}
+	" >> /etc/nginx/sites-available/${MAIN_SITE}
+			fi
+			ln -s /etc/nginx/sites-available/${MAIN_SITE} /etc/nginx/sites-enabled/${MAIN_SITE}
+			if [ "${CREATE_SSL}" = "yes" ]; then
+				NEWCERT ${NEW_SITE}
+			fi
+			
+			rm -f  ${FOLDER}/new-site.lastrun
+			daterun=`date +%Y-%m-%d-%H-%M-%S`
+			echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun
+			yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun
+			yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun
+			daterun=`date +%Y-%m-%d-%H-%M-%S`
+			echo -e "${daterun}" >> /etc/nginx/new-site.lastrun
+			DIVIDER true
+			echo -en "${idsCL[LightYellow]}Waiting for certificate replication between the nodes... ${idsCL[Default]}" 
+			checked=false
+			timeout=`date --date='2 minutes' +%s`
+			until [ "${checked}" = "" ]; do
+				tchecked=''
+				for nip in "${NODE_HOSTS[@]}"; do
+					if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
+						else NCMD="ssh root@${nip}"
+					fi
+					if [ "${NCMD}" != "" ]; then
+						tchecked+=`${NCMD} "cat /etc/nginx/new-site.lastrun" | diff - /etc/nginx/new-site.lastrun`
+						tchecked+=`${NCMD} "cat /var/www/new-site.lastrun" | diff - /var/www/new-site.lastrun`
+					fi
+				done
+				checked=${tchecked}
+				if [ "`date +%s`" -gt "$timeout" ]; then
+					echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
+					timeout=true
+					if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
+						echo "Timeout occurred in waiting for replication between nodes." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
+					fi
+					break
+				fi
+			done
+			if [ "${timeout}" != "true" ]; then
+				echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
+			fi
+			echo
+			
+			SERVICE nginx reload
+			echo ""
+			echo -e "${idsCL[LightGreen]}The new site for '${idsCL[LightGreen]}${NEW_SITE}${idsCL[Default]}' has been created.${idsCL[Default]}"
+			echo ""
+		else
+			echo "Missing proxy arguments"
+			Exit 1
+		fi
+
+	else
+		${SCRIPT} newsite
+		exit 0
+	fi
+}
+
+SET-PERMISSIONS(){
+	if [ "${1}" != "" ]; then
+		NEW_SITE=${1}
+		echo -e "${idsCL[Yellow]}Setting new site folder permissions for (/var/www/${NEW_SITE})${idsCL[Default]}"
+	fi
+	nid=1
+	for nip in "${NODE_HOSTS[@]}"
+	do
+		if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
+			nip='localhost '
+			NCMD=''
+		else
+			NCMD="ssh root@${nip}"
+		fi
+		if [ "${NEW_SITE}" != "" ]; then
+			echo -en "${idsCL[Cyan]}iDS-Node${nid} (${nip})    ${idsCL[Default]}"
+			${NCMD} chown -R www-data:www-data /var/www/$NEW_SITE
+			${NCMD} chmod -R 7775 /var/www/$NEW_SITE
+			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
+		else
+			echo -e "${idsCL[Yellow]}Setting folder permissions for iDS-Node${nid} (${nip})${idsCL[Default]}"
+			echo -en "${idsCL[Cyan]}LetsEncrypt Certs    ${idsCL[Default]}"
+			${NCMD} chown -R root:letsencrypt /etc/letsencrypt
+			${NCMD} chmod -R 6775 /etc/letsencrypt
+			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
+			echo -en "${idsCL[Cyan]}Webserver folders    ${idsCL[Default]}"
+			${NCMD} chown -R www-data:www-data /var/www
+			${NCMD} chmod -R 7775 /var/www
+			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
+        	echo -en "${idsCL[Cyan]}Gitea Folder         ${idsCL[Default]}"
+			${NCMD} chown -R git:git /var/lib/gitea
+			${NCMD} chmod -R 750 /var/lib/gitea
+			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
+			echo ""
+		fi
+		nid=`expr $nid + 1`
+	done
+	if [ "${NEW_SITE}" != "" ]; then echo ""; fi
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		ENTER2CONTINUE
+	fi
+}
+
+SERVICES(){
+	while :
+	do
+		echo ""
+		echo -e "${idsCL[Green]}Node Services${idsCL[Default]}"
+		DIVIDER true
+		s=1
+		for srvc in "${NODE_SERVICES[@]}"
+		do
+			echo " [${s}] ${NM_SERVICES[${srvc}]}"
+			s=`expr $s + 1`
+		done
+		echo ""
+		if [ -z $action ] || [ "${action}" = "gui" ]; then
+			echo " [B] Back"
+		fi
+		echo " [Q] Quit"
+		echo ""
+		echo -e -n "${idsCL[LightYellow]}Please select a service from above:${idsCL[Default]} "
+		read -n 1 selsrvc
+		selsrvc=`expr $selsrvc - 1`
+		echo ""
+		if [ -z ${NODE_SERVICES[${selsrvc}]} ] && [ "${selsrvc}" != "Q" ] && [ "${selsrvc}" != "q" ] && [ "${selsrvc}" != "B" ] && [ "${selsrvc}" != "b" ]; then
+			echo "Thats an invaild option,"
+			echo "please select a valid option only."
+			sleep 1
+			SERVICES
+			exit 0
+		elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
+			exit 0
+		elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
+			GUI
+		else
+			echo ""
+			echo -e "${idsCL[Green]}Node Service Selected: '${idsST[Bold]}${NM_SERVICES[${NODE_SERVICES[${selsrvc}]}]}${idsST[Reset]}${idsCL[Green]}'${idsCL[Default]}"
+			DIVIDER true
+			s=1
+			for srvcopt in "${NM_SRVCOPT[@]}"; do
+				echo " [${s}] ${srvcopt~}"
+				s=`expr $s + 1`
+			done
+			echo ""
+			if [ -z $action ] || [ "${action}" = "gui" ]; then
+				echo " [B] Back"
+			fi
+			echo " [Q] Quit"
+			echo ""
+			echo -e -n "${idsCL[LightYellow]}Please select an action to run from above:${idsCL[Default]} "
+			read -n 1 selsrvcopt
+			selsrvcopt=`expr $selsrvcopt - 1`
+			echo ""
+			if [ -z ${NM_SRVCOPT[${selsrvcopt}]} ] && [ "${selsrvcopt}" != "Q" ] && [ "${selsrvcopt}" != "q" ] && [ "${selsrvcopt}" != "B" ] && [ "${selsrvcopt}" != "b" ]; then
+				echo "Thats an invaild option,"
+				echo "please select a valid option only."
+				sleep 1
+				SERVICES
+				exit 0
+			elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
+				exit 0
+			elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
+				SERVICES
+				exit 0
+			else
+				echo
+				SERVICE ${NODE_SERVICES[${selsrvc}]} ${NM_SRVCOPT[${selsrvcopt}]}
+				echo ""
+				if [ -z $action ] || [ "${action}" = "gui" ]; then
+					DIVIDER
+					ENTER2CONTINUE
+				fi
+				break
+			fi
+		fi
+	done
+	
+}
+
+SERVICE(){
+	if [ "${NM_SERVICES[${1}]}" = "" ]; then
+		echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}"
+		exit 1	
+	fi
+	if [[ "start,stop,restart,reload,enable,disable" != *"${2}"* ]] && [ "${1}" != "daemon-reload" ]; then
+		echo -e "${idsCL[Red]}(${3}) is not an allowed service action.${idsCL[Default]}"
+		exit 1
+	fi
+	if [ "$2" = "start" ] &&  [ "$1" = "haproxy" ]; then TACT='restart'; else TACT="${2}"; fi
+		
+	echo -e "${idsCL[LightGreen]}[[${NM_SERVICES[${1}]} ${NM_SRVCOPTS[${2}]}ing]]${idsCL[Default]}"
+	echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
+	
+	nid=1
+	for nip in "${NODE_HOSTS[@]}"
+	do
+		if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
+			nip='localhost '
+			NCMD=''
+		else
+			NCMD="ssh root@${nip}"
+		fi
+		echo -en "${NM_SRVCOPTS[${2}]}ing on iDS-Node${nid} ($nip)... ${idsCL[Default]}"
+		$NCMD systemctl $2 $1
+		if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
+			echo -e "${idsCL[Green]}OK${idsCL[Default]}"
+		elif [ "$(${NCMD} systemctl is-active ${1})" != "active" ]; then
+			echo -e "${idsCL[Green]}OK${idsCL[Default]}"
+		elif [ "$2" = "stop" ]; then
+			echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
+		else
+			echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
+		fi
+		nid=`expr $nid + 1`
+	done
+	
+	echo ""
+	echo -e "${idsCL[Green]}${NM_SERVICES[${1}]} has been ${NM_SRVCOPTS[${2}]}ed${idsCL[Default]}"
+	echo ""
+}
+
+GUI(){
+	DISP_HEADER true true
+	while :
+	do
+		echo " [1] Node Status"
+		echo " [2] Node Services"
+		echo ""
+		echo " [3] New Site"
+		echo " [4] Delete Site"
+		echo ""
+		echo " [5] New Certificate"
+		echo " [6] Renew Certificates"
+		echo " [7] List Certificates"
+		echo ""
+		echo " [8] Set folder permissions on nodes"
+		echo " [9] Backup Node Settings and Files"
+		echo ""
+		echo " [Q] Quit"
+		echo ""
+		echo ""
+		echo -e -n "${idsCL[LightYellow]}Please select an [ActionItem] from above:${idsCL[Default]} "
+		read -n 1 opt
+		echo ""
+		case $opt in
+			1) STATUS; GUI;;
+			2) SERVICES; GUI;;
+			3) NEWSITE; GUI;;
+			4) DELSITES; GUI;;
+			5) NEWCERT; GUI;;
+			6) CERTRENEW; GUI;;
+			7) LISTCERTS; GUI;;
+			8) SET-PERMISSIONS; GUI;;
+			9) BACKUP; GUI;;
+			[Qq]) EXIT1
+				exit 0;;
+			*)	echo "Thats an invaild option,";
+				echo "please select a valid option only.";
+				sleep 1;;
+		esac
+	done
+}
+if [ ${action-x} ]; then
+	if [[ "${noheader}" != *" ${action} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
+		DISP_HEADER
+	fi
+	case $action in
+		newcert) NEWCERT ${2};;
+		certrenew) CERTRENEW;;
+		listcerts) LISTCERTS;;
+		nightlyrenew) NIGHTLYRENEW;;
+		delsite) DELSITE ${2} ${3} ${4} ${5};;
+		delsites) DELSITES;;
+		newsite) NEWSITE ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14};;
+		update) ;;
+		backup) BACKUP;;
+		status) STATUS ${2};;
+		status-check) STATUS-CHECK ${2};;
+		set-permissions) DIVIDER; SET-PERMISSIONS ${2};;
+		service) SERVICE ${2} ${3};;
+		services) SERVICES;;
+		gui) GUI;;
+		*)
+			echo -e "Usage: ${idsCL[Yellow]}nodemgmt${idsCL[Default]} { ${idsCL[Yellow]}service${idsCL[Default]} [service] [action] | ${idsCL[Yellow]}status${idsCL[Default]} | ${idsCL[Yellow]}newcert${idsCL[Default]} [domain(,s)] | ${idsCL[Yellow]}certrenew${idsCL[Default]} }"
+			echo ""
+			echo ""
+			exit 0;;
+	esac
+else
+	GUI
+fi
+
+exit 0
\ No newline at end of file
diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index 8e725b41..fc72ccf4 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -584,7 +584,6 @@ LISTCERTS(){
 	echo -e "Subject Name                  Expires       Alternate Subjects"
 	DIVIDER
 	for certdir in /etc/letsencrypt/live/*/ ; do
-		echo $certdir
 		SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
 		SUBJECTNAMES=$(openssl x509 -in ${certdir}/cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV)
 		SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, }
@@ -593,22 +592,6 @@ LISTCERTS(){
 		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g")
 		CERTEXPIRE=$(date -d "$(: | openssl x509 -in ${certdir}/cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s'); 
 		DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS)));
-		
-		AWK=$(which awk)
-		DATE=$(which date)
-		GREP=$(which grep)
-		OPENSSL=$(which openssl)
-		SED=$(which sed)
-
-		
-        #CERTDATE=$(${OPENSSL} x509 -in ${certdir}/cert.pem -text -enddate | grep 'Not After')
-		
-        # COMMONNAME=$(${OPENSSL} x509 -in ${certdir}/cert.pem -text -subject | \
-        #            ${SED} -e 's/.*CN=//' | \
-        #            ${SED} -e 's/\/.*//')
-					   
-		#SUBJECT=${COMMONNAME}
-
 
 		declare -i cw; declare -i spc1; declare -i c
 		c=0; cw=30; spc=''