#!/usr/bin/env bash
# NodeMgmt - Galera/NGINX Node Management Scripts

action="$1"

source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/nodemgmt/settings.conf
source /opt/idssys/nodemgmt/defaults.inc

#source /opt/idssys/nodemgmt/ssl-check.sh


# ========================================================= #

STATUS(){
	start=`date +%s`
	ST_ACTION=${1}
	if [ "${ST_ACTION}" = "report" ]; then
		unset idsCL idsBG idsST
		idsCL=('')
		idsBG=('')
		idsST=('')
	elif [ "${ST_ACTION}" = "sync" ]; then
		ST_ACTION=repl
	fi
	declare -i cw; declare -i spc1; declare -i c
	
	########################
	if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
		if [ "${ST_ACTION}" != "check" ]; then
			echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}"
		fi
		for NTYPE in "${NODE_TYPES[@]}"; do
			PH=${NTYPE}_HOSTS[0]
			if [ ! -f ${FOLDER}/${!PH}.down ]; then
				PH_CMD="ssh root@${!PH}"
				var=${NTYPE}_REPL_CHECK[@]
				if [ ! -z ${!var+x} ]; then
					for rcheck in "${!var}"; do
						# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
						${PH_CMD} rm -f ${FOLDER}/test.repl
						daterun=`date +%Y-%m-%d-%H-%M-%S`
						${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" &
					done
				fi
			fi
		done
		if [ "${ST_ACTION}" != "check" ]; then
			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
			echo
		fi
	fi

	########################
	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then
		if [ ! -z ${LOCAL_SERVICES+x} ]; then
			if [ "${ST_ACTION}" != "check" ]; then
				lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
				uptime=`uptime -p`
				echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
				DIVIDER false green
			fi
			for srvc in "${LOCAL_SERVICES[@]}"; do
				if [ "${ST_ACTION}" != "check" ]; then
					if [ "${ST_ACTION}" != "report" ]; then
						c=0; cw=30; spc=''
						spc1=${cw}-${#NM_SERVICES[${srvc}]}
						until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
					else
						spc=' '
					fi
					echo -en " ${NM_SERVICES[${srvc}]}${spc}: "
				fi
				
				if [ "$(systemctl is-active ${srvc})" != "active" ]; then
					if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
						if [ "${ST_ACTION}" != "check" ]; then
							echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
						fi
						SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1
						touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
					else
						errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
						if [ $errtime -gt ${RENOTIFY} ]; then
							if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
								mv ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
							fi
							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
							if [ "${ST_ACTION}" != "check" ]; then
								echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
							fi
							touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
						fi
					fi
				else
					if [ "${ST_ACTION}" != "check" ]; then
						echo -e "${idsCL[Green]}Running${idsCL[Default]}"
					fi
					if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
						if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
						else
							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
						fi
						SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.*
						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE}
						
					fi
				fi
			done
			
			if [ "${ST_ACTION}" != "check" ]; then echo; fi
		fi
		########################
		
		for NTYPE in "${NODE_TYPES[@]}"; do
			nid=1
			
			dockers=${NTYPE}_DOCKER[@]
			hosts=${NTYPE}_HOSTS[@]
			
			var=${NTYPE}_HOSTS[@]
			
			if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" == "dockers" ]; then
				GOFORCHECK=false;
			else
				GOFORCHECK=true;
			fi
				
			if [ ${GOFORCHECK} = true ]; then
			
				if [ "${ST_ACTION}" != "check" ]; then
					echo -e "${idsST[Bold]}"; DIVIDER
					if [ "${ST_ACTION}" = "dockers" ]; then
						echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
					else
						echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
					fi
					DIVIDER; echo -e "${idsST[Reset]}"
				fi
				for nip in "${!var}"; do
					[ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}"
					
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
						else NCMD="ssh root@${nip}"; LH=''
					fi
					if [ "${NCMD}" != "" ]; then
						checkhost=$(CHECK_HOST ${nip})
					fi
					if [ "${checkhost}" != "false" ]; then
						if [ "${ST_ACTION}" != "check" ]; then
							#echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}"
							if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" == "tst" ]; then
								uptime=`${NCMD} uptime -p`
								echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
							else
								echo -e "${idsCL[Default]}"
							fi
							
							DIVIDER false green
						fi
						if [ -f ${FOLDER}/${nip}.down ]; then
							if [ -f ${FOLDER}/${nip}.errtime ]; then
								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
							else
								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
							fi
							rm -f ${FOLDER}/${nip}.*
							SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
						fi
						
						if [ "${ST_ACTION}" != "dockers" ]; then
							srvcs=${NTYPE}_SERVICES_CHECK[@];
							srvcstotest="$(join_by " " ${!srvcs})"
							
							srvctst=(`${NCMD} systemctl is-active ${srvcstotest}`)
							sr=0
							for srvc in "${!srvcs}"; do
								[ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false || NOGOCHK=true
								if [ ${NOGOCHK} == true ]; then
									if [ "${ST_ACTION}" != "check" ]; then
										if [ "${ST_ACTION}" != "report" ]; then
											c=0; cw=30; spc=''; spc1=${cw}-${#NM_SERVICES[${srvc}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
										else
											spc=' '
										fi
										echo -en " ${NM_SERVICES[${srvc}]}$spc: "
									fi
									if [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" == "mysql" ]; then
										[ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false
									elif [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
										[ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false
									elif [ "${srvctst[$sr]}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then
										mysqlgo=true
									fi
									# echo "mysqlgo=${mysqlgo}"
									if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
										if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
											[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
											SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
											touch ${FOLDER}/${nip}-${srvc}.down
											echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
										else
											errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
											if [ $errtime -gt ${RENOTIFY} ]; then
												[ ! -f ${FOLDER}/${nip}-${srvc}.errtime ] && mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
												toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
												touch ${FOLDER}/${nip}-${srvc}.down
												# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
											fi
											[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
										fi
									else
										[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
										if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
											[ -f ${FOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
											SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
											echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}

											rm -f ${FOLDER}/${nip}-${srvc}.*

										fi
										# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
										if [ "${srvc}" == "gitea" ]; then
											rm -f ${FOLDER}/*-${srvc}.*
										fi
									fi
									[ "${mysqlgo}" = "true" ] && unset mysqlgo
									sr=`expr $sr + 1`
								fi
							done
							
							
							
							
							
							# for srvc in "${!var2}"; do
							# 	NOGOCHK=true;
							# 	[ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false;
							# 	if [ ${NOGOCHK} = true ]; then
							# 		if [ "${ST_ACTION}" != "check" ]; then
							# 			if [ "${ST_ACTION}" != "report" ]; then
							# 				c=0; cw=30; spc=''
							# 				spc1=${cw}-${#NM_SERVICES[${srvc}]}
							# 				until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
							# 			else
							# 				spc=' '
							# 			fi
							# 			echo -en " ${NM_SERVICES[${srvc}]}$spc: "
							# 		fi
							# 		srvctst=$(${NCMD} systemctl is-active ${srvc})
							# 		if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then
							# 			mysqlgo=false;
							# 			[ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true;
							# 		elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
							# 			mysqlgo=false;
							# 			[ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true;
							# 		elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then
							# 			mysqlgo=true
							# 		fi
							# 		# echo "mysqlgo=${mysqlgo}"
							# 		if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
							# 			if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
							# 					if [ "${ST_ACTION}" != "check" ]; then
							# 						echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
							# 					fi
							# 					SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
							# 					touch ${FOLDER}/${nip}-${srvc}.down
							# 					echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
							#
							# 			else
							# 				errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
							# 				if [ $errtime -gt ${RENOTIFY} ]; then
							# 					if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
							# 						mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
							# 					fi
							# 					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
							# 					touch ${FOLDER}/${nip}-${srvc}.down
							# 					if [ "${srvc}" = "mysql" ]; then
							# 						${NCMD} systemctl restart ${srvc}
							# 					fi
							# 				fi
							# 				if [ "${ST_ACTION}" != "check" ]; then
							# 					echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
							# 				fi
							# 			fi
							# 		else
							# 			if [ "${ST_ACTION}" != "check" ]; then
							# 				echo -e "${idsCL[Green]}Running${idsCL[Default]}"
							# 			fi
							# 			if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
							# 				if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
							# 					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
							# 				else
							# 					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
							# 				fi
							# 				SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
							# 				echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
							#
							# 				rm -f ${FOLDER}/${nip}-${srvc}.down
							# 				rm -f ${FOLDER}/${nip}-${srvc}.errtime
							#
							# 			fi
							# 			# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
							# 			if [ "${srvc}" == "gitea" ]; then
							# 				rm -f ${FOLDER}/*-${srvc}.down
							# 				rm -f ${FOLDER}/*-${srvc}.errtime
							# 			fi
							# 		fi
							# 		[ "${mysqlgo}" = "true" ] && unset mysqlgo
							# 	fi
							# done
						fi
						
						if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then
							if [ "${ST_ACTION}" != "check" ]; then
								echo
								echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
								echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}"
							fi
						
							for docker in "${!dockers}"; do
								if [ "${ST_ACTION}" != "check" ]; then
									if [ "${ST_ACTION}" != "report" ]; then
										c=0; cw=30; spc=''
										spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
										until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
									else
										spc=' '
									fi
									echo -en " ${NM_DOCKERS[${docker}]}$spc: "
								fi
								
								if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
									if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
										${NCMD} docker rm ${docker} >/dev/null 2>&1
									fi
									${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
									sleep 10s
									if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
										if [ "${ST_ACTION}" != "check" ]; then
											echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
										fi
										SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed"
										rm -f ${FOLDER}/${nip}-${docker}.down
										rm -f ${FOLDER}/${nip}-${docker}.errtime
										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE}

									else
										if [ "${ST_ACTION}" != "check" ]; then
											echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
										
											if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
												SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1
												touch ${FOLDER}/${nip}-${docker}.down
												echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE}
											else
												errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down)
												if [ $errtime -gt ${RENOTIFY} ]; then
													if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then
														mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime
													fi
													toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime)
													if [ "${ST_ACTION}" != "check" ]; then
														echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
													fi
													touch ${FOLDER}/${nip}-${docker}.down
												fi
											fi
										
										fi
									fi
								else
									if [ "${ST_ACTION}" != "check" ]; then
										echo -e "${idsCL[Green]}Running${idsCL[Default]}"
									fi
									rm -f ${FOLDER}/${nip}-${docker}.down
									rm -f ${FOLDER}/${nip}-${docker}.errtime
								fi
							done
						fi
					
					else
						if [ ! -f ${FOLDER}/${nip}.down ]; then
							touch ${FOLDER}/${nip}.down
							if [ ! -f ${FOLDER}/${nip}.errtime ]; then
								touch ${FOLDER}/${nip}.errtime
							fi
							if [ "${ST_ACTION}" != "check" ]; then
								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
								echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
							fi
							SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1
							echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE}
						else
							if [ "${ST_ACTION}" != "check" ]; then
								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
								echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
							fi
							errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
							# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then
							# 	echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}
							# fi
						fi

					fi
					if [ "${ST_ACTION}" != "check" ]; then echo; fi
					nid=`expr $nid + 1`
				done

			fi #GOFORCHECK
		
		done
		
	fi
	########################
	## REPLICATION CHECK
	########################
	
	if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
		if [ "${ST_ACTION}" != "check" ]; then
			echo -e "${idsST[Bold]}"; DIVIDER
			echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
			DIVIDER; echo -e "${idsST[Reset]}"
		fi
	
		for NTYPE in "${NODE_TYPES[@]}"; do
			PH=${NTYPE}_HOSTS[0]
			PH_CMD="ssh root@${!PH}"
			var=${NTYPE}_REPL_CHECK[@]
			if [ ! -z ${!var+x} ]; then
				nid=1
			
				var=${NTYPE}_HOSTS[@]
				for nip in "${!var}"; do
				
					# echo "CHECK: ${NTYPE} - ${nip}"
				
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
						if [ "${ST_ACTION}" != "check" ]; then
							echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
							DIVIDER false green
						fi
						
						# checkhost=$(CHECK_HOST ${nip})
						#if [ "${checkhost}" != "false" ]; then
						if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
							var2=${NTYPE}_REPL_CHECK[@]
							# timeout=`date --date='20 seconds' +%s`
							for rcheck in "${!var2}"; do
								if [ "${ST_ACTION}" != "check" ]; then
									if [ "${ST_ACTION}" != "report" ]; then
										c=0; cw=30; spc=''
										spc1=${cw}-${#REPL_DESC[${rcheck}]}
										until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
									else
										spc=' '
									fi
									echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
									
									timeout=`date --date='2 minutes' +%s`
									
									# if [ "${rcheck}" == "www" ]; then
									#  	timeout=`date --date='2 minutes' +%s`
									#  else
									# 	timeout=`date --date='20 seconds' +%s`
									#  fi
								
								else
									timeout=`date --date='2 minutes' +%s`
								fi
								warn_timeout=`date --date='30 secs' +%s`
								
								checked=false
								until [ "${checked}" = "" ]; do
									if [ "${PH_CMD}" = "" ]; then
										ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
									else
										ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
									fi
									if [ "`date +%s`" -gt "${timeout}" ]; then
										timeout=true
										break
									elif [ "`date +%s`" -gt "${warn_timeout}" ]; then
										echo -en " waiting for sync...  "
										warn_timeout=`date --date='100 minutes' +%s`
									fi
								done
								if [ "${timeout}" = "true" ]; then
									if [ "${ST_ACTION}" != "check" ]; then
										echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
									fi
									if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
										touch ${FOLDER}/${nip}-${rcheck}.down
										SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
									fi
								else
									if [ "${ST_ACTION}" != "check" ]; then
										echo -e "${idsCL[Green]}Good${idsCL[Default]}"
									fi
									if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
										rm -f ${FOLDER}/${nip}-${rcheck}.down
										SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
									fi
								fi
				
							done
						else
							if [ "${ST_ACTION}" != "check" ]; then
								[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}"
								[ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}"
							fi
						fi
						if [ "${ST_ACTION}" != "check" ]; then echo; fi
					fi
					nid=`expr $nid + 1`
				done
			fi
		done
		##########################
		# REMOVE REPL CHECK FILES
		##########################
		if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
			if [ "${ST_ACTION}" != "check" ]; then
				echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
			fi
			#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
			#LBHCMD="ssh root@${LB_HOSTS[0]}"
			#WHCMD="ssh root@${WEB_HOSTS[0]}"
			for NTYPE in "${NODE_TYPES[@]}"; do
				PH=${NTYPE}_HOSTS[0]
				if [ ! -f ${FOLDER}/${!PH}.down ]; then
					PH_CMD="ssh root@${!PH}"
					var=${NTYPE}_REPL_CHECK[@]
					if [ ! -z ${!var+x} ]; then
						for rcheck in "${!var}"; do
							${PH_CMD} rm -f ${FOLDER}/test.repl
							daterun=`date +%Y-%m-%d-%H-%M-%S`
							if [ "${PH_CMD}" = "" ]; then
								rm -f ${FOLDER}/test.repl &
								rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
							else
								${PH_CMD} rm -f ${FOLDER}/test.repl &
								${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
							fi
						done
					fi
				fi
			done
			rm -Rf ${FOLDER}/test.repl
			if [ "${ST_ACTION}" != "check" ]; then
				echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
				echo
			fi
		fi	
	fi
	
	########################
	## FREE SPACE CHECK
	########################
	#if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then
	if [ "${ST_ACTION}" = "freespace" ]; then
		if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then
			for NTYPE in "${NODE_TYPES[@]}"; do
				if [ "${ST_ACTION}" != "check" ]; then
					echo -e "${idsST[Bold]}"; DIVIDER
					echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Free Space Scan${idsCL[Default]}"
					DIVIDER; echo -e "${idsST[Reset]}"
				fi
				nid=1
				var=${NTYPE}_HOSTS[@]
				for nip in "${!var}"; do
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
						else NCMD="ssh root@${nip}"; LH=''
					fi
					if [ "${ST_ACTION}" != "check" ]; then
						echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
						DIVIDER false green
					fi
					
					[ "${ST_ACTION}" != "check" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
					declare -A partitions
					DRIVEINFO=$(ssh root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
					DRIVEINFO=(${DRIVEINFO})
					echo -en "\e[1A";
					echo -e "\e[0K\r"
					
					NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
					for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
						ii=$((${i}*3))
						pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
						pname=${pname#*vg-}
						if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then
							freespace=${DRIVEINFO[$((${ii}+2))]//M/}
							if [ "${ST_ACTION}" != "check" ]; then
								c=0; cw=20; spc=''
								spc1=${cw}-${#pname}
								until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
								echo -en " ${pname}$spc: "
							fi
							if [[ ${freespace} -gt 1024 ]]; then
								fsgb=$(bc <<< "scale=2; ${freespace}/1024")
								fsdsp="${fsgb} GB"
							else
								fsdsp="${freespace} MB"
							fi
							if [ "${freespace}" -le "1024" ]; then
								fs_status='error'
								fs_status_color='Red'
								SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free" 1
							elif [ "${freespace}" -le "5120" ]; then
								fs_status='warn'
								fs_status_color='Yellow'
								SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free"
							else
								fs_status=''
								fs_status_color='Green'
							fi
							[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
						fi
					done
					
					# for partition in "${partitions3[@]}"; do
					# 	if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then
					# 		pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
					# 		pname=${pname#*vg-}
					# 		if [ "${ST_ACTION}" != "check" ]; then
					# 			c=0; cw=20; spc=''
					# 			spc1=${cw}-${#pname}
					# 			until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
					# 			echo -en " ${pname}$spc: "
					# 		fi
					#
					# 		# [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition}
					# 		# freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1
					#
					# 		if [ "${freespace}" -le "1000" ]; then
					# 			fs_status='error'
					# 			fs_status_color='Red'
					# 			SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" 1
					#
					# 		elif [ "${freespace}" -le "5000" ]; then
					# 			fs_status='warn'
					# 			fs_status_color='Yellow'
					# 			SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free"
					#
					# 		else
					# 			fs_status=''
					# 			fs_status_color='Green'
					#
					# 		fi
					# 		if [ "${ST_ACTION}" != "check" ]; then
					# 			if [[ ${freespace} -gt 1000 ]]; then
					# 				fsgb=$(bc <<< "scale=2; ${freespace}/1000")
					# 				fsdsp="${fsgb} GB"
					# 			else
					# 				fsdsp="${freespace} MB"
					# 			fi
					# 			echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
					#
					# 		fi
					# 	fi
					# done

					nid=`expr $nid + 1`
					echo
				done
			done
		fi
	fi
	########################
	########################
	
	if [ "${ST_ACTION}" != "check" ]; then
		echo ""
		if [ -z $action ] || [ "${action}" = "gui" ]; then
			DIVIDER true
			ENTER2CONTINUE
		fi
	fi
	end=`date +%s`
	runtime=$((end-start))
	echo "runtime: ${runtime}"
}

CHECK_DOCKER_SERVICES(){
	
	for NTYPE in "${NODE_TYPES[@]}"; do
		dockers=${NTYPE}_DOCKER[@]
		hosts=${NTYPE}_HOSTS[@]
		
		if [[ -v ${NTYPE}_DOCKER ]]; then
			if [ "${ST_ACTION}" != "check" ]; then
				echo -e "${idsST[Bold]}"; DIVIDER
				echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
				DIVIDER; echo -e "${idsST[Reset]}"
			fi
			nid=1
		
			for nip in "${!hosts}"; do
				if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
					else NCMD="ssh root@${nip}"; LH=''
				fi
				if [ "${NCMD}" != "" ]; then
					checkhost=$(CHECK_HOST ${nip})
				fi
				if [ "${checkhost}" != "false" ]; then
					if [ "${ST_ACTION}" != "check" ]; then
						echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}"
						uptime=`${NCMD} uptime -p`
						echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
						DIVIDER false green
					fi
					if [ -f ${FOLDER}/${nip}.down ]; then
						if [ -f ${FOLDER}/${nip}.errtime ]; then
							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
						else
							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
						fi
						rm -f ${FOLDER}/${nip}.down
						rm -f ${FOLDER}/${nip}.errtime
						SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
					fi

					for docker in "${!dockers}"; do
						if [ "${ST_ACTION}" != "check" ]; then
							c=0; cw=30; spc=''
							spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
							until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
							echo -en " ${NM_DOCKERS[${docker}]}$spc: "
						fi
						if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
						    if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
						        ${NCMD} docker rm ${docker}
						    fi
						    ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d
							sleep 10s
							if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
								if [ "${ST_ACTION}" != "check" ]; then
									echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
								fi
							else
								if [ "${ST_ACTION}" != "check" ]; then
									echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
								fi
							fi
						else
							if [ "${ST_ACTION}" != "check" ]; then
								echo -e "${idsCL[Green]}Running${idsCL[Default]}"
							fi
						fi
					done
				else
					if [ ! -f ${FOLDER}/${nip}.down ]; then
						touch ${FOLDER}/${nip}.down
						if [ ! -f ${FOLDER}/${nip}.errtime ]; then
							touch ${FOLDER}/${nip}.errtime
						fi
					else
						errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
					fi
					if [ "${ST_ACTION}" != "check" ]; then
						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
						echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
					fi
				fi
				if [ "${ST_ACTION}" != "check" ]; then echo; fi
				nid=`expr $nid + 1`
			done
		fi
	done

}

NEWCERT(){
	echo
	if [ -z ${1+x} ]; then
		echo -e -n "${idsCL[LightCyan]}Create certificate for what name (comma seperated for mutiple) : ${idsCL[Default]}"
		read NEW_CERT
		echo
	else
		NEW_CERT=${1}
	fi
	if [[ ${NEW_CERT} == *","* ]]; then
		IFS=','; NEW_CERTS=(${NEW_CERT}); unset IFS
		MAIN_CERT=${NEW_CERTS[0]}
	else
		MAIN_CERT=${NEW_CERT}
	fi
	if [ "${NEW_CERT}" != "" ]; then
		echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}"
		echo ""
		
		# echo -en "${idsCL[LightYellow]}Stopping other Webservers... ${idsCL[Default]}"
		# for nip in "${WEB_HOSTS[@]}"; do
		# 	if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
		# 		# `ssh root@${nip} service nginx stop`
		# 		# SERVICE nginx stop ${nip}
		# 	fi
		# done
		# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
		# echo
		
		$CERT_DAEMON certonly --webroot -w /opt/lb-data/letsencrypt-acme-challenge -d ${NEW_CERT}
		# $CERT_DAEMON certonly --dry-run --webroot -w /var/www/html -d ${NEW_CERT}
		
		chown -R root:le /opt/lb-data/letsencrypt
		chmod -R 6775 /opt/lb-data/letsencrypt
		
		if [ -d /opt/lb-data/letsencrypt/live/${MAIN_CERT} ]; then

			touch /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert
			if [ -f /opt/lb-data/nginx/sites-enabled/${MAIN_CERT}.conf ]; then
				rm -f  ${FOLDER}/cert-request.lastrun
				daterun=`date +%Y-%m-%d-%H-%M-%S`
				echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun
				yes | cp -rfH ${FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun
				DIVIDER true
				echo -e -n "${idsCL[LightCyan]}Reload NGINX on LB Nodes (Y/n): ${idsCL[Default]}"
				read -n 1 NGINXRELOAD
				if [[ ${NGINXRELOAD} =~ ^[Nn]$ ]]; then
					tmp=''
				else
					echo
					echo -en "${idsCL[LightYellow]}Waiting for Cert replication across the nodes... ${idsCL[Default]}"
					for nip in "${LB_HOSTS[@]}"; do
						checkhost=$(CHECK_HOST ${nip})
						if [ "${checkhost}" != "false" ]; then
							if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
								checked=false
								until [ "${checked}" = "" ]; do
									checked=`ssh root@${nip} "[ ! -f /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert ] && echo '.'"`
								done
							fi
						fi
					done
					rm -f /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert
					echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
				
					echo
					SERVICE nginx reload
				fi
			fi
			echo
			echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}"
		else
			echo
			echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}"
		fi
		
		# echo
		# echo -en "${idsCL[LightYellow]}Starting other Webservers... ${idsCL[Default]}"
		# for nip in "${WEB_HOSTS[@]}"; do
		# 	if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
		# 		`ssh root@${nip} service nginx start`
		# 		# SERVICE nginx start ${nip}
		# 	fi
		# done
		# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
		
		echo ""
		if [ -z $action ] || [ "${action}" = "gui" ]; then
			DIVIDER true
			ENTER2CONTINUE
		fi
	else
		echo "You havent entered a site address."
		sleep 1
		NEWCERT
		exit 0
	fi
}

DEL-SSL(){
	if [ -z ${1+x} ]; then
		echo -e -n "${idsCL[LightCyan]}Delete what SSL site address: ${idsCL[Default]}"
		read DEL_SSL
		echo ""
	else
		DEL_SSL=${1}
	fi
	if [ ! -z ${DEL_SSL+x} ] && [ "${DEL_SSL}" != "" ]; then
		if [ -d /etc/letsencrypt/live/${DEL_SSL} ]; then 
			echo -e "${idsCL[LightRed]}Deleting the SSL certificates for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}'...${idsCL[Default]}"
			echo ""
	
			echo -en "${idsCL[LightRed]}Removing Files and Folders... ${idsCL[Default]}"
			rm -rf /etc/letsencrypt/archive/${DEL_SSL}
			rm -rf /etc/letsencrypt/live/${DEL_SSL}
			rm -f /etc/letsencrypt/renewal/${DEL_SSL}.conf
			echo -e "${idsCL[Green]}OK${idsCL[Default]}"
			echo ""
		
			
		else
			echo -e "${idsCL[LightRed]}The SSL files folder for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}' could not be found.${idsCL[Default]}"
			exit 1
		fi
		if [ "${timeout}" != "true" ]; then
			echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
		fi
		echo
		# SERVICE nginx reload
		echo -e "${idsCL[LightRed]}The SSL certificate has been removed fromt be nodes.${idsCL[Default]}"
	fi
}

CERTRENEW(){
	echo -en "${idsCL[LightCyan]}Stopping Webserver-Node2...${idsCL[Default]}"
	ssh root@webserver-node2.scity.us service nginx stop
	echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}"
	echo
	echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}"
	echo
	sleep 5
	$CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
	# $CERT_DAEMON renew --force-renewal  --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
    # $CERT_DAEMON --dry-run  --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
	CONCAT_SSL
	chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
	chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
	yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
	daterun=`date +%Y-%m-%d-%H-%M-%S`
	echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
	DIVIDER true
	echo -en "${idsCL[LightCyan]}Starting Webserver-Node2 Back up...${idsCL[Default]}"
	ssh root@webserver-node2.scity.us service nginx start
	echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}"
	echo
	SERVICE nginx reload 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
	echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}"
	echo ""
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
		ENTER2CONTINUE
	fi
}
NIGHTLYRENEW(){
	rm -f ${FOLDER}/cert-renewal.lastrun
	ssh root@webserver-node2.scity.us service nginx stop
	sleep 5
	$CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun
	CONCAT_SSL
	chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
	chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
	yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
	daterun=`date +%Y-%m-%d-%H-%M-%S`
	echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
	ssh root@webserver-node2.scity.us service nginx start
	SERVICE nginx reload web &>> ${FOLDER}/cert-renewal.lastrun
}

CONCAT_SSL(){
	rm -f /tmp/ssllist
	for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist
	for certdir in $(</tmp/ssllist); do
		rm -f ${certdir}fullcert.pem
		cat ${certdir}privkey.pem ${certdir}fullchain.pem > ${certdir}fullcert.pem
	done
}

LISTCERTS(){
	declare -i cw; declare -i spc1; declare -i c
	declare -A CHECKCERT_DOMAINS
	IFS=$'\n'
	for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
	    HOST=${LINE%% *}
	    PORT=${LINE#* }
	    IFS=" "
		CHECKCERT_DOMAINS[${HOST}]=${PORT}
	done
	unset IFS
	if [ ! -z ${LOCAL_SERVICES+x} ]; then
		NCMD="ssh root@${MYSQL_HOSTS[0]}"
		#${NCMD} rm -f /tmp/ssllist
		#${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist
	else
		NCMD=''
		rm -f /tmp/ssllist
		#for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist
	fi
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
	fi
	echo 
	echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}"
	DIVIDER false yellow 120
	echo -e "Subject Name               Monitored   Expires      Alternate Subject Names"
	DIVIDER false yellow 120
	
	cw=30; 
	c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
	
	#ssldir=$(${NCMD} find /opt/nginx-proxy/ssl/* -type l)
	ssldir=$(${NCMD} find /opt/lb-data/letsencrypt/live/* -type d)
	for certdir in ${ssldir[@]}; do
		SUBJECT=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
		SUBJECTNAMES=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV)
		CERTEXPIRE=$(date -d "$(: | ${NCMD} openssl x509 -in ${certdir}/cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s');

		SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, }
		# SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/\n/, /g")
		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g")
		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g")
		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g")
		IFS=', '; SUBJECT_NAMES=(${SUBJECTNAMES}); unset IFS
		DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS)));
		
		c=0; spc=''
		spc1=${cw}-${#SUBJECT}
		until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
		if [ $DUEIN -gt $CERTEXPIRE ]; then
			date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}"
		else
			date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}"
		fi
		
		if [ "${CHECKCERT_DOMAINS[${SUBJECT}]}" = "" ]; then
			monitored='No '
		else
			monitored="${idsCL[Green]}Yes${idsCL[Default]}"
		fi
		if [ ${#SUBJECT_NAMES[@]} -lt 4 ]; then
			echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored}     ${date}    ${SUBJECTNAMES}"
		elif [ ${#SUBJECT_NAMES[@]} -lt 7 ]; then
			echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored}     ${date}    ${SUBJECT_NAMES[0]}, ${SUBJECT_NAMES[1]}, ${SUBJECT_NAMES[2]}"
			echo -e "${spc2}                      ${SUBJECT_NAMES[3]}, ${SUBJECT_NAMES[4]}, ${SUBJECT_NAMES[5]}"

		fi
		DIVIDER false darkGray 120
	done
	echo 
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
		ENTER2CONTINUE
	fi
	echo ""
}

LISTCERTS_NPM(){
	declare -i cw; declare -i spc1; declare -i c
	declare -A CHECKCERT_DOMAINS
	IFS=$'\n'
	for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
	    HOST=${LINE%% *}
	    PORT=${LINE#* }
	    IFS=" "
		CHECKCERT_DOMAINS[${HOST}]=${PORT}
	done
	unset IFS
	if [ ! -z ${LOCAL_SERVICES+x} ]; then
		NCMD="ssh root@${MYSQL_HOSTS[0]}"
		${NCMD} rm -f /tmp/ssllist
		${NCMD} 'for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist
	else
		NCMD=''
		rm -f /tmp/ssllist
		for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist
	fi
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
	fi
	echo 
	echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}"
	DIVIDER false yellow 120
	echo -e "Subject Name               Monitored   Expires      Alternate Subject Names"
	DIVIDER false yellow 120
	
	cw=30; 
	c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
	for certdir in $(</tmp/ssllist); do
		SUBJECT=$(${NCMD} openssl x509 -in ${certdir}cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
		SUBJECTNAMES=$(${NCMD} openssl x509 -in ${certdir}cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV)
		CERTEXPIRE=$(date -d "$(: | ${NCMD} openssl x509 -in ${certdir}cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s');

		SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, }
		# SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/\n/, /g")
		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g")
		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g")
		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g")
		IFS=', '; SUBJECT_NAMES=(${SUBJECTNAMES}); unset IFS
		DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS)));
		
		c=0; spc=''
		spc1=${cw}-${#SUBJECT}
		until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
		if [ $DUEIN -gt $CERTEXPIRE ]; then
			date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}"
		else
			date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}"
		fi
		
		if [ "${CHECKCERT_DOMAINS[${SUBJECT}]}" = "" ]; then
			monitored='No '
		else
			monitored="${idsCL[Green]}Yes${idsCL[Default]}"
		fi
		if [ ${#SUBJECT_NAMES[@]} -lt 4 ]; then
			echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored}     ${date}    ${SUBJECTNAMES}"
		elif [ ${#SUBJECT_NAMES[@]} -lt 7 ]; then
			echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored}     ${date}    ${SUBJECT_NAMES[0]}, ${SUBJECT_NAMES[1]}, ${SUBJECT_NAMES[2]}"
			echo -e "${spc2}                      ${SUBJECT_NAMES[3]}, ${SUBJECT_NAMES[4]}, ${SUBJECT_NAMES[5]}"

		fi
		DIVIDER false darkGray 120
	done
	echo 
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
		ENTER2CONTINUE
	fi
	echo ""
}
COPYCERTS_NPM(){
	ssldir='/opt/nginx-proxy/ssl'
	
	if [ "${1}" = "reset" ]; then
		echo "Resetting ... Deleting all SSL symlink folders"
		ssldirs=$(find ${ssldir} -type l)
		for certdir in ${ssldirs[@]}; do
			rm ${certdir}
		done
	else
		baddirs=$(find $ssldir -type l ! -exec test -e {} \; -print)
		for od in ${baddirs[@]}; do
			echo "Removing bad symlink folder ... ${od}"
			rm -f ${od}
		done
	fi
	
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
	fi
	
	ssls=$(find /opt/nginx-proxy/letsencrypt/live/* -type d)
	for certdir in ${ssls[@]}; do
		SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
		if [ ! -L ${ssldir}/${SUBJECT} ]; then
			echo "${SUBJECT} ... creating"
			/bin/ln -s ${certdir} ${ssldir}/${SUBJECT}
		fi
	done
	
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		echo
		DIVIDER true
		ENTER2CONTINUE
		echo
	fi
}

CHECK_NPMCERTS(){
	ssls=$(find /opt/nginx-proxy/letsencrypt/live/* -type d)
	
	for certdir in ${ssls[@]}; do
		SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
		ENDDATE=$(openssl x509 -in ${certdir}/cert.pem -noout -enddate | awk -F '=' '{print $2}')
		DAYS=$(dateutils.ddiff today "$(date --date="${ENDDATE}" '+%F')" -f '%d')
		
		c=0; cw=35; spc=''
		spc1=`expr ${cw} - ${#SUBJECT}`
		until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
		[ ${DAYS} -lt 0 ] && spc=${spc::-1}
		
		if [ ${DAYS} -lt 24 ]; then
			echo "${SUBJECT}:${spc}${DAYS} days remaining"
		fi
		
	done
	
	echo
	
}

CHECK-CERTS(){
	if [ "${1}" == "check" ]; then
		rm -f ${FOLDER}/ssl-cert-check.lastrun
		${FOLDER}/ssl-cert-check/ssl-cert-check -a -f ${FOLDER}/ssl-cert-check/ssldomains > ${FOLDER}/ssl-cert-check.lastrun
		SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)"
		SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG"
		
	else
		${FOLDER}/ssl-cert-check/ssl-cert-check -a -f ${FOLDER}/ssl-cert-check/ssldomains
	fi
}



BACKUP(){
	BACKUP_FOLDER=/opt/idssys/backups/node-backup
	#BACKUP_FOLDER=/opt/idssys/backups/node-backups/${NODE_HOSTNAME} #/`date +%Y-%m-%d`
	#BACKUP_TIMEFOLDER=${BACKUP_DAYFOLDER}/`date +%H-%M`
	if [ "${action}" != "backup" ]; then
		echo -e "${idsCL[Green]}Backing up Node Settings and Files...${idsCL[Default]}"
		DIVIDER true
	fi
	if [ ! -d ${BACKUP_FOLDER} ]; then
		mkdir -p ${BACKUP_FOLDER}
	fi
	for item in "${!BACKUP_ITEMS[@]}"
	do
		if [ "${action}" != "backup" ]; then
			echo -en "Backing up '${item}'... "
		fi
		#tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${BACKUP_ITEMS[$item]} .
		tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -g ${BACKUP_FOLDER}/backup.snar  -C ${BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git'
		if [ "${action}" != "backup" ]; then
			echo -e "${idsCL[Green]}Ok${idsCL[Default]}"
		fi
	done
	touch ${FOLDER}/backup.lastrun
	if [ "${action}" != "backup" ]; then
		echo ""
		DIVIDER true
	fi
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		ENTER2CONTINUE
	fi
}

DELSITE(){
	while [ $# -gt 0 ]; do
	    case "$1" in
	        -site) DEL_SITE=${2};;
	        -ssl) DEL_SSL=${2};;
			-list) DELSITES; exit 0;;
	   	 	-*)
				echo "Invalid option: '${1}' requires an argument" 1>&2
				echo ""
				echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
				width=33
				printf "%-${width}s- %s\n" "    -site {FQDN address}" "(*required)"
				printf "%-${width}s- %s\n" "    -ssl {yes or [no]}" "Delete SSL certs as well"
				printf "%-${width}s- %s\n" "    -list" "List sites (same as running nodemgmt delsites)"
				echo "}"
				exit 1;;
	    esac
		shift
	done
	if [ -z ${DEL_SITE+x} ]; then
		echo -e -n "${idsCL[LightCyan]}Delete what site address: ${idsCL[Default]}"
		read DEL_SITE
		echo ""
	fi
	if [[ $DEL_SSL =~ ^[Nn]$ ]]; then
		DEL_SSL=no
	elif [[ $DEL_SSL =~ ^[Yy]$ ]]; then
		DEL_SSL=yes
	elif [ -z ${DEL_SSL+x} ]; then
		echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${DEL_SITE}' as well? [y/N]${idsCL[Default]} "
		read DEL_SSL
	fi
	if [ ! -z ${DEL_SITE+x} ] && [ "${DEL_SITE}" != "" ]; then		
		echo -e "${idsCL[LightRed]}Deleting site '${idsCL[Red]}${DEL_SITE^^}${idsCL[LightRed]}'...${idsCL[Default]}"
		echo ""
	
		echo -e "${idsCL[LightRed]}[[Removing Files and Folders]]${idsCL[Default]}"
		echo -e "${idsCL[LightRed]}-------------------------------------------${idsCL[Default]}"
		echo
		echo -en "${idsCL[LightCyan]}Removing files from all Nodes ... ${idsCL[Default]}"
		ssh root@10.10.1.120 rm -f /etc/nginx/sites-enabled/${DEL_SITE}* >/dev/null 2>&1
		ssh root@10.10.10.80 rm -f /etc/nginx/sites-enabled/${DEL_SITE}* >/dev/null 2>&1
		if [ "${DEL_SSL}" == "yes" ]; then
			ssh root@10.10.10.80 rm -rf /etc/letsencrypt/archive/${DEL_SITE} >/dev/null 2>&1
			ssh root@10.10.10.80 rm -rf /etc/letsencrypt/live/${DEL_SITE} >/dev/null 2>&1
			ssh root@10.10.10.80 rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf >/dev/null 2>&1
		fi
		echo -e "${idsCL[Green]}Done${idsCL[Default]}"
		echo
		echo -en "${idsCL[LightCyan]}Reloading NGINX ... ${idsCL[Default]}"
		ssh root@10.10.10.80 nodemgmt service nginx reload >/dev/null 2>&1
		ssh root@10.10.1.120 nodemgmt service nginx reload >/dev/null 2>&1
		echo -e "${idsCL[Green]}Done${idsCL[Default]}"
		
		# nid=1
		# for nip in "${WEB_HOSTS[@]}"; do
		# 	if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
		# 		nip='localhost '
		# 		NCMD=''
		# 	else
		# 		NCMD="ssh root@${nip}"
		# 	fi
		# 	echo -en "Removing from Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
		# 	if [ "${NCMD}" != "" ]; then
		# 		checkhost=$(CHECK_HOST ${nip})
		# 	fi
		# 	if [ "${checkhost}" != "false" ]; then
		# 		#if [ -f /etc/nginx/sites-available/${DEL_SITE} ]; then
		# 			${NCMD} rm -f /etc/nginx/sites-available/${DEL_SITE}*
		# 			${NCMD} rm -f /etc/nginx/sites-enabled/${DEL_SITE}*
		# 			#fi
		# 		#if [ -d /var/www/${DEL_SITE} ]; then
		# 			${NCMD} rm -rf /var/www/${DEL_SITE}
		# 			#fi
		# 		if [ "${DEL_SSL}" = "yes" ]; then
		# 			${NCMD} rm -rf /etc/letsencrypt/archive/${DEL_SITE}
		# 			${NCMD} rm -rf /etc/letsencrypt/live/${DEL_SITE}
		# 			${NCMD} rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf
		# 		fi
		# 		echo -e "${idsCL[Green]}OK${idsCL[Default]}"
		# 	else
		# 		echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
		# 	fi
		# 	nid=`expr $nid + 1`
		# done
		# echo ""
		# SERVICE nginx reload ns
		echo -e "${idsCL[LightRed]}Site has been deleted.${idsCL[Default]}"
	else
		echo "Missing arguments"
		echo ""
		echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
		width=33
		printf "%-${width}s- %s\n" "    -site {FQDN address}" "Site to delete"
		printf "%-${width}s- %s\n" "    -ssl {yes or [no]}" "Delete SSL certs as well"
		printf "%-${width}s- %s\n" "    -list" "List sites (same as running nodemgmt delsites)"
		echo "}"
		exit 1
	fi
}

DELSITES(){
	echo
	echo -e "${idsCL[Red]}Select a site to delete...${idsCL[Default]}"
	DIVIDER true
	sid=1
	filels="( $(ssh root@${WEB_HOSTS[0]} ls '/etc/nginx/sites-available/*') )"
	# IFS='\n'
	for siteconf in $filels; do
	# for siteconf in /etc/nginx/sites-available/* ; do
		# [ -e "$siteconf" ] || continue
		if [ ${siteconf:0:1} == '/' ]; then
			IFS='/'; site_conf=(${siteconf}); unset IFS
			SITES[${sid}]=${site_conf[4]}
			sid=`expr $sid + 1`
		fi
	done
	for s in "${!SITES[@]}"; do
		echo " [${s}] ${SITES[${s}]}"
	done
	echo ""
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		echo " [B] Back"
	fi
	echo " [Q] Quit"
	echo ""
	echo -e -n "${idsCL[LightYellow]}Please select a site from above from above:${idsCL[Default]} "
	read selsite
	echo ""
	if [ -z ${SITES[$selsite]} ] && [ "${selsite}" != "Q" ] && [ "${selsite}" != "q" ] && [ "${selsite}" != "B" ] && [ "${selsite}" != "b" ]; then
		echo "Thats an invaild option,"
		echo "please select a valid option only."
		sleep 1
		DELSITES
		exit 0
	elif [ "${selsite}" = "Q" ] || [ "${selsite}" = "q" ]; then
		exit 0
	elif [ "${selsite}" = "B" ] || [ "${selsite}" = "b" ]; then
		GUI
	else
		while :
		do
			echo -e -n "${idsCL[LightRed]}Are you sure you want to delete '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}'? [y/N]${idsCL[Default]} "
			read response
			echo ""
			if [[ $response =~ ^[Yy]$ ]]; then
				echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}', if they exist? [y/N]${idsCL[Default]} "
				read sslresponse
				DELSITE -site ${SITES[${selsite}]} -ssl ${sslresponse}
				echo ""
				DIVIDER
				ENTER2CONTINUE
				break
			else
				break
			fi			
		done
		DELSITES
		exit 0
	fi
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		ENTER2CONTINUE
	fi
}

NEWSITE(){
	echo
	while [ $# -gt 0 ]; do
		case "$1" in
			-site) NEW_SITE=${2};;
			-type) SITE_TYPE=${2};;
			-ssl) CREATE_SSL=${2};;
			-proxy_scheme) PROXYSCHEME=${2};;
			-proxy_host) PROXYHOST=${2};;
			-proxy_port) PROXYPORT=${2};;
			-websocket) WEBSOCKET=${2};;
			-hsts) HSTS=${2};;
			-exploits) EXPLOITS=${2};;
			-secure) SECURE=${2};;
			-h | -help | --help)
				echo ""
				echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {"
				width=33
				printf "%-${width}s- %s\n" "    -site {FQDN address(,es)}" "(new site and aliases, comma separated)"
				printf "%-${width}s- %s\n" "    -ssl {yes or no}" "(defaults to yes)"
				printf "%-${width}s- %s\n" "    -type {'local' or 'proxy'}" "(defaults to local)"
				printf "%-${width}s- %s\n" "    -proxy_port {host port}" "(proxy backend host)"
				printf "%-${width}s- %s\n" "    -proxy_host {IP or FQDN}" "(proxy backend port)"
				printf "%-${width}s- %s\n" "    -proxy_scheme {http or https}" "(proxy backend scheme)"
				printf "%-${width}s- %s\n" "    -websocket {yes or no}" "(websocket support)"
				printf "%-${width}s- %s\n" "    -hsts {yes or no}" "(hsts support)"
				printf "%-${width}s- %s\n" "    -exploits {yes or no}" "(block exploits)"
				printf "%-${width}s- %s\n" "    -secure {yes or no}" "(secure access [nginx/.htpasswd])"
				echo "}"
				exit 1;;
		esac
		shift
	done
		
	#if [ -z ${SITE_TYPE+x} ]; then SITE_TYPE=local; fi
	#if [ -z ${CREATE_SSL+x} ]; then CREATE_SSL=true; fi
	if [ -z ${NEW_SITE+x} ]; then
		echo -e -n "${idsCL[LightCyan]}New site domain name (comma seperated for multiple) : ${idsCL[Default]}"
		read NEW_SITE
		showdivide=yes
	fi
	
	if [ -z ${CREATE_SSL+x} ]; then
		echo -e -n "${idsCL[LightCyan]}Create SSL for site? [Y/n] ${idsCL[Default]}"
		read CREATE_SSL
		showdivide=yes
		if [[ $CREATE_SSL =~ ^[Yy]$ ]] || [ "${CREATE_SSL}" = "" ]; then
			CREATE_SSL=yes
			# echo -e -n "${idsCL[LightCyan]}Add additonal domain names to the SSL cert (comma seperated)? : ${idsCL[Default]}"
			# read ssladd
		else
			CREATE_SSL=no
		fi
	fi

	if [ -z ${SITE_TYPE+x} ]; then
		echo -e -n "${idsCL[LightCyan]}Site type (local/{proxy}): ${idsCL[Default]}"
		read SITE_TYPE
		showdivide=yes
		if [ "${SITE_TYPE}" = "" ]; then
			SITE_TYPE=proxy
		fi
	fi
	if [ "${SITE_TYPE}" = "proxy" ]; then
		if [ -z ${PROXYHOST+x} ]; then
			echo -e -n "${idsCL[LightCyan]}What is the proxy backend address (IP or FQDN): ${idsCL[Default]}"
			read PROXYHOST
			showdivide=yes
		fi
		if [ -z ${PROXYPORT+x} ]; then
			echo -e -n "${idsCL[LightCyan]}What is the proxy backend port (tcp port): ${idsCL[Default]}"
			read PROXYPORT
			showdivide=yes
		fi
		if [ -z ${PROXYSCHEME+x} ]; then
			echo -e -n "${idsCL[LightCyan]}What is the proxy backend scheme (http/https): ${idsCL[Default]}"
			read PROXYSCHEME
			showdivide=yes
		fi
		if [ -z ${WEBSOCKET+x} ]; then
			echo -e -n "${idsCL[LightCyan]}Enable Websocket Support (y/N): ${idsCL[Default]}"
			read WEBSOCKET
			showdivide=yes
			if [[ ${WEBSOCKET} =~ ^[Nn]$ ]] || [ "${WEBSOCKET}" = "" ]; then
				WEBSOCKET=no
			elif [[ ${WEBSOCKET} =~ ^[Yy]$ ]]; then
				WEBSOCKET=yes
			else
				WEBSOCKET=no
			fi
		fi
		if [ -z ${HSTS+x} ]; then
			echo -e -n "${idsCL[LightCyan]}Enable HSTS Support (Y/n): ${idsCL[Default]}"
			read HSTS
			showdivide=yes
			[[ ${HSTS} =~ ^[Yy]$ ]] || [ "${HSTS}" = "" ] &&HSTS=yes || HSTS=no
		fi
		if [ -z ${EXPLOITS+x} ]; then
			echo -e -n "${idsCL[LightCyan]}Block exploits (y/N): ${idsCL[Default]}"
			read EXPLOITS
			showdivide=yes
			if [[ ${EXPLOITS} =~ ^[Nn]$ ]] || [ "${EXPLOITS}" = "" ]; then
				EXPLOITS=no
			elif [[ ${EXPLOITS} =~ ^[Yy]$ ]]; then
				EXPLOITS=yes
			else
				EXPLOITS=no
			fi
		fi
		if [ -z ${SECURE+x} ]; then
			echo -e -n "${idsCL[LightCyan]}Secure site with passwords from [ nginx/.htpasswd ](y/N): ${idsCL[Default]}"
			read SECURE
			showdivide=yes
			if [[ ${SECURE} =~ ^[Nn]$ ]] || [ "${SECURE}" = "" ]; then
				SECURE=no
			elif [[ ${SECURE} =~ ^[Yy]$ ]]; then
				SECURE=yes
			else
				SECURE=no
			fi
		fi
	fi
	[ "${showdivide}" == "yes" ] && DIVIDER
	echo
	width=14
	printf "%-${width}s: %s\n" "New site" "${NEW_SITE}"
	printf "%-${width}s: %s\n" "Create SSL" "${CREATE_SSL}"
	printf "%-${width}s: %s\n" "Site type" "${SITE_TYPE}"
	if [ "${SITE_TYPE}" = "proxy" ]; then
		printf "%-${width}s: %s\n" "Proxy host" "${PROXYHOST}"
		printf "%-${width}s: %s\n" "Proxy port" "${PROXYPORT}"
		printf "%-${width}s: %s\n" "Proxy scheme" "${PROXYSCHEME}"
		printf "%-${width}s: %s\n" "Websocket Support" "${WEBSOCKET}"
		printf "%-${width}s: %s\n" "HSTS Support" "${HSTS}"
		printf "%-${width}s: %s\n" "Block Exploits" "${EXPLOITS}"
		printf "%-${width}s: %s\n" "Secure Access" "${SECURE}"
	fi
	echo -e -n "${idsCL[LightRed]}Is this information correct? [Y/n]${idsCL[Default]} "
	read -n 1 response
	echo
	if [[ $response =~ ^[Yy]$ ]] || [ "${response}" = "" ]; then 
		if [[ ${NEW_SITE} == *","* ]]; then
			IFS=','; NEW_SITES=(${NEW_SITE}); unset IFS
			MAIN_SITE=${NEW_SITES[0]}
			NGINX_SERVERNAME=${NEW_SITE//[,]/ }
		else
			MAIN_SITE=${NEW_SITE}
			NGINX_SERVERNAME=${NEW_SITE}
				
		fi
		if [ "${SITE_TYPE}" = "proxy" ]; then
			if [ ! -z ${PROXYSCHEME+x} ] && [ ! -z ${PROXYHOST+x} ] && [ ! -z ${PROXYPORT+x} ]; then GO=true; fi
			else GO=true
		fi
		if [ "${GO}" = "true" ]; then
			echo -e "${idsCL[LightGreen]}Setting up new site for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}"
			echo ""
			
			[ "${WEBSOCKET}" == "yes" ] && WEBSOCKET="include conf.d\/include\/websocket-support.conf;" || WEBSOCKET=""
			[ "${HSTS}" == "yes" ] && HSTS="include conf.d\/include\/hsts-support.conf;" || HSTS=""
			[ "${EXPLOITS}" == "yes" ] && EXPLOITS="include conf.d\/include\/block-exploits.conf;" || EXPLOITS=""
			[ "${SECURE}" == "yes" ] && SECURE="include conf.d\/include\/secure-access.conf;" || SECURE=""
			
			
			######################################### LOCAL
			if [ "${SITE_TYPE}" = "local" ]; then
				echo -e "server {
	listen 80;" > /etc/nginx/sites-available/${MAIN_SITE}
				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "	listen 443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "
	server_name ${NGINX_SERVERNAME};
	
	set \$base /var/www/${MAIN_SITE};
	root \$base/public_html;

	access_log  /var/log/nginx/${MAIN_SITE}-access.log;
	error_log  /var/log/nginx/${MAIN_SITE}-error.log warn;" >> /etc/nginx/sites-available/${MAIN_SITE}

				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "
	ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem;
	ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem;
	include conf.d/include/ssl-ciphers.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "
	index index.php;

	location / {
		try_files \$uri \$uri/ /index.php?\$query_string;" >> /etc/nginx/sites-available/${MAIN_SITE}
				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "		include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "	}

	location ~ \.php\$ {
		fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
		include conf.d/include/php_fastcgi.conf;
	}

	include conf.d/include/general.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
				if [ "${CREATE_SSL}" = "yes" ]; then
					echo -e "	include conf.d/include/letsencrypt-acme-challenge.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
				fi
				echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE}
				

				sudo -u www-data mkdir -p /var/www/${MAIN_SITE}/{public_html,nginx_logs}
				# echo -en "${idsCL[LightYellow]}Waiting for folder replication across the webserver nodes... ${idsCL[Default]}"
				# for nip in "${WEB_HOSTS[@]}"; do
				# 	checkhost=$(CHECK_HOST ${nip})
				# 	if [ "${checkhost}" != "false" ]; then
				# 		if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
				# 			checked=false
				# 			until [ "${checked}" = "" ]; do
				# 				checked=`ssh root@${nip} "[ ! -d /var/www/${MAIN_SITE} ] && echo does not exist"`
				# 			done
				# 		fi
				# 	fi
				# done
				# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
				#
				# echo -en "${idsCL[LightYellow]}Setting folder permissions... ${idsCL[Default]}"
				# SET-PERMISSIONS ${MAIN_SITE}
				# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
				
				
							
			######################################### PROXY
			else
				
				cp ${FOLDER}/templates/nginx.proxy.site /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				sed -i "s/<<SERVER_NAME>>/${NGINX_SERVERNAME}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				sed -i "s/<<MAIN_SITE>>/${MAIN_SITE}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				sed -i "s/<<PROXY_IP>>/${PROXYHOST}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				sed -i "s/<<PROXY_PORT>>/${PROXYPORT}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				sed -i "s/<<PROXY_SCHEME>>/${PROXYSCHEME}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				sed -i "s/<<WEBSOCKET>>/${WEBSOCKET}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				sed -i "s/<<HSTS>>/${HSTS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				sed -i "s/<<EXPLOITS>>/${EXPLOITS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				sed -i "s/<<SECURE>>/${SECURE}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
			fi
			
			if [ "${CREATE_SSL}" = "yes" ]; then
				[ "${ssladd}" != "" ] && NEWCERT ${NEW_SITE},${ssladd} || NEWCERT ${NEW_SITE}
				if [ "${SITE_TYPE}" == "proxy" ]; then
					sed -i "s/#ssl_certificate/ssl_certificate/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
				fi
			fi
			
			rm -f  ${FOLDER}/new-site.lastrun
			daterun=`date +%Y-%m-%d-%H-%M-%S`
			echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun
			# yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun
			# yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun
			# daterun=`date +%Y-%m-%d-%H-%M-%S`
			# echo -e "${daterun}" >> /etc/nginx/new-site.lastrun
			DIVIDER true
			echo ""
			echo -e "${idsCL[LightGreen]}The new site for '${idsCL[LightGreen]}${NEW_SITE}${idsCL[Default]}' has been created.${idsCL[Default]}"
			echo ""
		else
			echo "Missing proxy arguments"
			Exit 1
		fi

	else
		${SCRIPT} newsite
		exit 0
	fi
}

SET-PERMISSIONS(){
	if [ "${1}" = "gitea" ]; then
		nid=1
		for nip in "${WEB_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			if [ "${2}" != "q" ]; then
				echo -en "${idsCL[Yellow]}Resetting folder permissions for Gitea on Webserver-Node${nid} (${nip})... ${idsCL[Default]}"
			fi
			checkhost=$(CHECK_HOST ${nip})
			if [ "${checkhost}" != "false" ]; then
				${NCMD} chown -R git:git /var/lib/gitea
				${NCMD} chmod -R 750 /var/lib/gitea
				${NCMD} chown -R git:git /home/git
				${NCMD} chmod -R 750 /home/git
				if [ "${2}" != "q" ]; then
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
				fi
			elif [ "${2}" != "q" ]; then
				echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
			fi
			nid=`expr $nid + 1`
		done
	elif [ "${1}" = "pdnsadmin" ]; then
		nid=1
		for nip in "${WEB_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			if [ "${2}" != "q" ]; then
				echo -en "${idsCL[Yellow]}Resetting folder permissions for PowerDNS-Admin on Webserver-Node${nid} (${nip})... ${idsCL[Default]}"
			fi
			checkhost=$(CHECK_HOST ${nip})
			if [ "${checkhost}" != "false" ]; then
				${NCMD} chown -R www-data:www-data /opt/powerdns-admin
				${NCMD} chmod -R 775 /opt/powerdns-admin
				if [ "${2}" != "q" ]; then
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
				fi
			elif [ "${2}" != "q" ]; then
				echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
			fi
			nid=`expr $nid + 1`
		done
	elif [ "${1}" = "nextcloud" ]; then
		nip=${NC_HOSTS[0]}
		if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
			NCMD=''
		else
			NCMD="ssh root@${nip}"
		fi
		if [ "${2}" != "q" ]; then
			echo -en "${idsCL[Yellow]}Resetting folder permissions for Nextcloud on Nextcloud-Node1 (${nip})... ${idsCL[Default]}"
		fi
		checkhost=$(CHECK_HOST ${nip})
		if [ "${checkhost}" != "false" ]; then
			${NCMD} chown -R www-data:www-data /mnt/Nextcloud-Data
			${NCMD} find /mnt/Nextcloud-Data/ -type d -exec chmod 770 {} \;
			${NCMD} find /mnt/Nextcloud-Data/ -type f -exec chmod 660 {} \;
			if [ "${2}" != "q" ]; then
				echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
			fi
		elif [ "${2}" != "q" ]; then
			echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
		fi
	else
		DIVIDER
		if [ "${1}" != "" ]; then
			NEW_SITE=${1}
			echo -e "${idsCL[Yellow]}Setting new site folder permissions for (/var/www/${NEW_SITE})${idsCL[Default]}"
		fi
		if [ "${NEW_SITE}" = "xxx" ]; then
			nid=1
			for nip in "${MYSQL_HOSTS[@]}"; do
				checkhost=$(CHECK_HOST ${nip})
				if [ "${checkhost}" != "false" ]; then
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
						nip='localhost '
						NCMD=''
					else
						NCMD="ssh root@${nip}"
					fi
					echo -e "${idsCL[Yellow]}Setting folder permissions for MySQL-Node${nid} (${nip})${idsCL[Default]}"
					echo -en "${idsCL[Cyan]}LetsEncrypt Certs    ${idsCL[Default]}"
					${NCMD} chown -R root:letsencrypt /etc/letsencrypt
					${NCMD} chmod -R 6775 /etc/letsencrypt
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
					echo ""
				fi
				nid=`expr $nid + 1`
			done
		fi
		nid=1
		for nip in "${WEB_HOSTS[@]}"; do
			checkhost=$(CHECK_HOST ${nip})
			if [ "${checkhost}" != "false" ]; then
				if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
					nip='localhost '
					NCMD=''
				else
					NCMD="ssh root@${nip}"
				fi
				if [ "${NEW_SITE}" != "" ]; then
					echo -en "${idsCL[Cyan]}Webserver-Node${nid} (${nip})    ${idsCL[Default]}"
					${NCMD} chown -R www-data:www-data /var/www/$NEW_SITE
					${NCMD} chmod -R 7775 /var/www/$NEW_SITE
					${NCMD} chown -R www-data:www-data /var/lib/php/sessions
					${NCMD} chmod -R 7777 /var/lib/php/sessions
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
				else
					echo -e "${idsCL[Yellow]}Setting folder permissions for Webserver-Node${nid} (${nip})${idsCL[Default]}"
					echo -en "${idsCL[Cyan]}Webserver folders    ${idsCL[Default]}"
					${NCMD} chown -R www-data:www-data /var/www
					${NCMD} chmod -R 7775 /var/www
					${NCMD} chown -R www-data:www-data /var/lib/php/sessions
					${NCMD} chmod -R 7777 /var/lib/php/sessions
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
		        	echo -en "${idsCL[Cyan]}Gitea Folder         ${idsCL[Default]}"
					${NCMD} chown -R git:git /var/lib/gitea
					${NCMD} chmod -R 750 /var/lib/gitea
					${NCMD} chown -R git:git /home/git
					${NCMD} chmod -R 750 /home/git
					echo -e "${idsCL[Green]}Complete${idsCL[Def ault]}"
					echo -en "${idsCL[Cyan]}LetsEncrypt Certs    ${idsCL[Default]}"
					${NCMD} chown -R root:letsencrypt /etc/letsencrypt
					${NCMD} chmod -R 6775 /etc/letsencrypt
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
					echo -en "${idsCL[Cyan]}PowerDNS-Admin       ${idsCL[Default]}"
					${NCMD} chown -R www-data:www-data /opt/powerdns-admin
					${NCMD} chmod -R 775 /opt/powerdns-admin
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
					echo ""
				fi
			fi
			nid=`expr $nid + 1`
		done
		if [ "${NEW_SITE}" != "" ]; then echo ""; fi
		if [ -z $action ] || [ "${action}" = "gui" ]; then
			ENTER2CONTINUE
		fi
	fi
}

SERVICES(){
	while :
	do
		echo ""
		echo -e "${idsCL[Green]}Node Services${idsCL[Default]}"
		DIVIDER true
		s=1
		for srvc in "${NODE_SERVICES[@]}"
		do
			echo " [${s}] ${NM_SERVICES[${srvc}]}"
			s=`expr $s + 1`
		done
		echo ""
		if [ -z $action ] || [ "${action}" = "gui" ]; then
			echo " [B] Back"
		fi
		echo " [Q] Quit"
		echo ""
		echo -e -n "${idsCL[LightYellow]}Please select a service from above:${idsCL[Default]} "
		read -n 1 selsrvc
		selsrvc=`expr $selsrvc - 1`
		echo ""
		if [ -z ${NODE_SERVICES[${selsrvc}]} ] && [ "${selsrvc}" != "Q" ] && [ "${selsrvc}" != "q" ] && [ "${selsrvc}" != "B" ] && [ "${selsrvc}" != "b" ]; then
			echo "Thats an invaild option,"
			echo "please select a valid option only."
			sleep 1
			SERVICES
			exit 0
		elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
			exit 0
		elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
			GUI
		else
			echo ""
			echo -e "${idsCL[Green]}Node Service Selected: '${idsST[Bold]}${NM_SERVICES[${NODE_SERVICES[${selsrvc}]}]}${idsST[Reset]}${idsCL[Green]}'${idsCL[Default]}"
			DIVIDER true
			s=1
			for srvcopt in "${NM_SRVCOPT[@]}"; do
				echo " [${s}] ${srvcopt~}"
				s=`expr $s + 1`
			done
			echo ""
			if [ -z $action ] || [ "${action}" = "gui" ]; then
				echo " [B] Back"
			fi
			echo " [Q] Quit"
			echo ""
			echo -e -n "${idsCL[LightYellow]}Please select an action to run from above:${idsCL[Default]} "
			read -n 1 selsrvcopt
			selsrvcopt=`expr $selsrvcopt - 1`
			echo ""
			if [ -z ${NM_SRVCOPT[${selsrvcopt}]} ] && [ "${selsrvcopt}" != "Q" ] && [ "${selsrvcopt}" != "q" ] && [ "${selsrvcopt}" != "B" ] && [ "${selsrvcopt}" != "b" ]; then
				echo "Thats an invaild option,"
				echo "please select a valid option only."
				sleep 1
				SERVICES
				exit 0
			elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
				exit 0
			elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
				SERVICES
				exit 0
			else
				echo
				SERVICE ${NODE_SERVICES[${selsrvc}]} ${NM_SRVCOPT[${selsrvcopt}]}
				echo ""
				if [ -z $action ] || [ "${action}" = "gui" ]; then
					DIVIDER
					ENTER2CONTINUE
				fi
				break
			fi
		fi
	done
	
}

SERVICE(){
	NODE_TYPE=''
	#var=NC_HOSTS[@]
	
	if [ "$1" = "glusterd" ]; then
		
		for nip in "${GL_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi

			if [ "${3}" != "q" ]; then
				echo -en "${NM_SRVCOPTS[${2}]}ing on '$nip'... ${idsCL[Default]}"
			fi
			checkhost=$(CHECK_HOST ${nip})
			if [ "${checkhost}" != "false" ]; then
				#$NCMD systemctl $2 $1 > /dev/null 2>&1
				if [ "${3}" != "q" ]; then
					if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
						echo -e "${idsCL[Green]}OK${idsCL[Default]}"
					elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
						echo -e "${idsCL[Green]}OK${idsCL[Default]}"
					elif [ "$2" = "stop" ]; then
						echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
					else
						echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
					fi
				fi
			elif [ "${3}" != "q" ]; then
				echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
			fi

		done
		
		if [ "${2}" = "start" ] || [ "${2}" = "restart" ]; then
			
			echo
			sleep 5s
			
			for nip in "${GL_HOSTS[@]}"; do
				if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
					nip='localhost '
					NCMD=''
				else
					NCMD="ssh root@${nip}"
				fi
			
				if [ "${3}" != "q" ]; then
					echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
				fi
				checkhost=$(CHECK_HOST ${nip})
				if [ "${checkhost}" != "false" ]; then
					#$NCMD mount -a > /dev/null 2>&1
					if [ "${3}" != "q" ]; then
						if [[ "${NCHOSTS}" == *"${nip}"* ]]; then
							mntpath="/mnt/Nextcloud-Data"
						elif [[ "${WEBHOSTS}" == *"${nip}"* ]]; then
							mntpath="/mnt/web-data"
						fi
						${NCMD} "test -e ${mntpath}/mounted"
						if [ $? -eq 0 ]; then
							echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
						else
							echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
						fi
					fi
					
					if [ "$nip" = "10.10.1.43" ]; then
						echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
						mntpath="/mnt/Nextcloud-Data"
						${NCMD} "test -e ${mntpath}/mounted"
						if [ $? -eq 0 ]; then
							echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
						else
							echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
						fi
					fi
					
				elif [ "${3}" != "q" ]; then
					echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
				fi
			
			done
			
		fi
		
	else
		if [ "${4}" != "all" ]; then
			IP=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
			for NTYPE in "${NODE_TYPES[@]}"; do
				var=${NTYPE}_HOSTS[@]
				for nip in "${!var}"; do
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
						NODE_TYPE=${NTYPE};
						break 2
					fi
				done
			done
			if [ "$NODE_TYPE" = "NC" ]; then
				NT_HOSTS=NC_HOSTS[@]
				NT_HOST=NC_HOSTS[0]
				NTS=('NC'); #NTS=$NTS1[@]
			elif [ "$NODE_TYPE" = "LB" ]; then
				NT_HOSTS=LB_HOSTS[@]
				NT_HOST=LB_HOSTS[0]
				NTS=('LB'); #NTS=$NTS1[@]
			elif [ "$NODE_TYPE" = "WEB" ]; then
				NT_HOSTS=WEB_HOSTS[@]
				NT_HOST=WEB_HOSTS[0]
				NTS=('WEB'); #NTS=$NTS1[@]
			elif [ "$NODE_TYPE" = "MYSQL" ]; then
				NT_HOSTS=MYSQL_HOSTS[@]
				NT_HOST=MYSQL_HOSTS[0]
				NTS=('MYSQL'); #NTS=$NTS1[@]
			elif [ "${1}" = "nginx" ]; then
				NT_HOSTS=WEB_HOSTS[@]
				NT_HOST=WEB_HOSTS[0]
				NTS=('WEB'); #NTS=NTS1[@]
				# NTS=$NODE_TYPES
			else
				NTS=(${NODE_TYPES[*]})
			fi
			
		else
			NTS=(${NODE_TYPES[*]})
		fi
	
		if [ "${3}" != "q" ]; then
			if [ "${NM_SERVICES[${1}]}" = "" ]; then
				echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}"
				exit 1	
			fi
			if [ "${NM_SRVCOPTS[${2}]}" = "" ]; then
				echo -e "${idsCL[Red]}(${2}) is not an allowed service action.${idsCL[Default]}"
				exit 1
			fi
			if [ "$2" = "start" ] &&  [ "$1" = "haproxy" ]; then
				TACT='restart'
			else
				TACT="${2}"
			fi

			echo
			echo -e "${idsCL[LightGreen]}[[ ${NM_SERVICES[${1}]} ${NM_SRVCOPTS[${2}]}ing ]]${idsCL[Default]}"
			echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
		fi
	

		if [ "${1}" = "nginx-x" ]; then
			if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
				if [ "${3}" != "ns" ]; then
					if [ "${3}" != "q" ]; then
						echo -en "${idsCL[LightYellow]}Verifying nginx config and ssl cert replication across the nodes... ${idsCL[Default]}"
					fi
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${!NT_HOST}"* ]]; then
						PH_CMD="ssh root@localhost"
					else
						PH_CMD="ssh root@${!NT_HOST}"
					fi
					${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/nginx/test.repl"
					# if [ "$NODE_TYPE" != "NC" ]; then
					# 	${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/letsencrypt/test.repl"
					# fi
					for nip in "${!NT_HOSTS}"; do
						checkhost=$(CHECK_HOST ${nip})
						if [ "${checkhost}" != "false" ]; then
							if [ "${nip}" != "${!NT_HOST}" ]; then
								checked=false
								until [ "${checked}" = "" ]; do
									checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/nginx/test.repl ]; then cat /etc/nginx/test.repl; fi' | diff - /etc/nginx/test.repl\"`"
									# checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/nginx/test.repl ]; then cat /etc/letsencrypt/test.repl; fi' | diff - /etc/letsencrypt/test.repl\"`"
								done
							fi
						fi
					done
					${PH_CMD} rm -f /etc/nginx/test.repl
					# if [ "$NODE_TYPE" != "NC" ]; then
					# 	${PH_CMD} rm -f /etc/letsencrypt/test.repl
					# fi
				fi
			fi
		
		elif [ "${1}" = "apache2-x" ]; then
			if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
				if [ "${3}" != "ns" ]; then
					if [ "${3}" != "q" ]; then
						echo -en "${idsCL[LightYellow]}Verifying Apache config and ssl cert replication across the nodes... ${idsCL[Default]}"
					fi
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${!NT_HOST}"* ]]; then
						PH_CMD="ssh root@localhost"
					else
						PH_CMD="ssh root@${!NT_HOST}"
					fi
					echo "HERE: '$PH_CMD'"

					${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/apache2/test.repl"
					for nip in "${!NT_HOSTS}"; do
						checkhost=$(CHECK_HOST ${nip})
						if [ "${checkhost}" != "false" ]; then
							if [ "${nip}" != "${!NT_HOST}" ]; then
								checked=false
								until [ "${checked}" = "" ]; do
									checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/apache2/test.repl ]; then cat /etc/apache2/test.repl; fi' | diff - /etc/apache2/test.repl\"`"
								done
							fi
						fi
					done
					${PH_CMD} rm -f /etc/apache2/test.repl
				fi
			fi
		fi
		if [ "${3}" != "q" ]; then
			echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
			echo
		fi
	
		for NTYPE in "${NTS[@]}"; do
			var1=${NTYPE}SERVICES_CHECK
			if [[ "${!var1}" = *"${1}"* ]]; then
				nid=1
				if [ "${3}" != "" ]; then
					var2=(${3})
					sethost=true;
				else
					var2=${NTYPE}_HOSTS[@]
					IFS=' '
					var2=(${!var2})
					unset IFS
					sethost=false
				fi
				
				for nip in "${var2[@]}"; do
					# if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${3}"* ]]; then GO=true;
					# elif [ ! -z ${3+x} ] || [ "${3}" == "q" ]; then GO=true;
					# else GO=false;
					# fi
					# if [ "${GO}" == "true" ]; then
					if [ $sethost == true ]; then
						nodename=${NM_HOSTNAMES[${3}]}
					else
						nodename="${NM_NODETYPES[$NTYPE]}-Node${nid}"
					fi
						if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
							nip='localhost '
							NCMD=''
						else
							NCMD="ssh root@${nip}"
						fi
					
						NOGOCHK=true;
						# if [ "${1}" == "gitea" ] || [ "${1}" == "pdnsadmin" ] || [ "${1}" == "pdnsadmin.socket" ]; then
						if [ "${1}" == "gitea" ]; then
							if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.10.1.120* ]]; then
								NOGOCHK=false;
							fi
						fi
					
						if [ "${NOGOCHK}" == "true" ]; then
							if [ "${3}" != "q" ]; then
								echo -en "${NM_SRVCOPTS[${2}]}ing on ${nodename} ($nip)... ${idsCL[Default]}"
							fi
							checkhost=$(CHECK_HOST ${nip})
							if [ "${checkhost}" != "false" ]; then
								$NCMD systemctl $2 $1 >/dev/null 2>&1
								if [ "${3}" != "q" ]; then
									if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
										echo -e "${idsCL[Green]}OK${idsCL[Default]}"
									elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
										echo -e "${idsCL[Green]}OK${idsCL[Default]}"
									elif [ "$2" = "stop" ]; then
										echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
									else
										echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
									fi
								fi
							elif [ "${3}" != "q" ]; then
								echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
							fi
						fi
					# fi
					nid=`expr $nid + 1`
					# if [ "${1}" = "gitea" ]; then
					# 	break
					# fi
				done
			fi
		done
	fi
	
	if [ "${3}" != "q" ]; then
		echo
		echo -e "${idsCL[Green]}${NM_SERVICES[${1}]} has been ${NM_SRVCOPTS[${2}]}ed${idsCL[Default]}"
		echo
	fi
}

HAPROXY-CONFIG() {
	echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}"
	DIVIDER false yellow
	if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then
		nid=1
		for nip in "${MYSQL_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on MySQL-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
		echo
	fi
	if [[ "${WEBSERVICES_CHECK}" = *"haproxy"* ]]; then
		nid=1
		for nip in "${WEB_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
	fi
	if [[ "${LBSERVICES_CHECK}" = *"haproxy"* ]]; then
		nid=1
		for nip in "${LB_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on LB-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
	fi
	SERVICE haproxy reload
	echo ""
	echo -e "${idsCL[Green]}HAProxy Config Files have been created.${idsCL[Default]}"
	echo ""
}

SINGLE_SERVER_SERVICE_CHECK(){
	if [ "${1}" = "web" ]; then
		# servicestocheck=gitea,pdnsadmin
		servicestocheck=gitea
		IFS=,; services_to_check=(${servicestocheck}); unset IFS
		
		local_ips=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1);
		local_ip=$(/sbin/ip -o -4 addr list eth0 |grep 255 | awk '{print $4}' | cut -d/ -f1)
		if [[ ${local_ips} = *10.10.1.120* ]]; then
			if [ -f /mnt/web-data/mounted ]; then
				for stc in "${services_to_check[@]}"; do
					if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
						SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
						echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${LOGFILE}
						
						[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
						for nip in "${WEB_HOSTS[@]}"; do
							if [[ "${local_ip}" != *"${nip}"* ]]; then
								ssh root@${nip} /bin/systemctl stop ${stc}
							fi
						done
						sleep 3s
						/bin/systemctl start ${stc} &
					fi
				done
				for nip in "${WEB_HOSTS[@]}"; do
					if [[ "${local_ip}" != *"${nip}"* ]]; then
						ssh root@${nip} /usr/bin/docker stop vaultwarden >/dev/null 2>&1
					fi
				done
				/usr/bin/docker start vaultwarden & >/dev/null 2>&1
			fi

		else
			for stc in "${services_to_check[@]}"; do
				[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
				/bin/systemctl stop ${stc} &
			done
			/usr/bin/docker stop vaultwarden & >/dev/null 2>&1
		fi
	
		if [ "$(systemctl is-active nginx)" != "active" ]; then
			/usr/local/bin/nodemgmt service web restart ${local_ip}
		fi
	fi
}

KEEPALIVE-CONFIG() {
	echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"
	DIVIDER false yellow
	if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then
		nid=1
		for nip in "${MYSQL_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on MySQL-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/keepalived/keepalived.conf"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
		echo
	fi
	if [[ "${WEBSERVICES_CHECK}" = *"keepalive"* ]]; then
		nid=1
		for nip in "${WEB_HOSTS[@]}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/keepalived/keepalived.conf"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
	fi
	SERVICE keepalived reload
	echo ""
	echo -e "${idsCL[Green]}Keepalive Config Files have been created.${idsCL[Default]}"
	echo ""
}

NODEUPDATE() {
	start=`date +%s`
	if [ "${1}" = "report" ]; then
		unset idsCL idsBG idsST
		idsCL=('')
		idsBG=('')
		idsST=('')
	fi
	declare -i cw; declare -i spc1; declare -i c
	echo -e "${idsCL[LightGreen]}Updating All Nodes${idsCL[Default]}"
	echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
	nip=${MYSQLMANAGER}
	if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
		nip='localhost'
	fi
	c=0; cw=14; spca=''
	spc1=${cw}-7
	until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
	c=0; cw=15; spcb=''
	spc1=${cw}-${#nip}
	until [ $c = ${spc1} ]; do spcb="${spcb} "; c=`expr $c + 1`; done
	echo -en "Updating MySQL-Manager${spca}($nip)${spcb}...  ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
	ssh root@${nip} "nodemgmt update q"
	echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]}  ...  ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
	ssh root@${nip} "runup runup"
	echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
	echo
	for NTYPE in "${NODE_TYPES[@]}"; do
		nid=1
		var=${NTYPE}_HOSTS[@]
		for nip in "${!var}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
				nip='localhost  '
			fi
			NCMD="ssh root@${nip}"
			c=0; cw=14; spca=''
			spc1=${cw}-${#NM_NODETYPES[$NTYPE]}
			until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
			c=0; cw=15; spcb=''
			spc1=${cw}-${#nip}
			until [ $c = ${spc1} ]; do spcb="${spcb} "; c=`expr $c + 1`; done
			echo -en "Updating ${NM_NODETYPES[$NTYPE]}-Node${nid}${spca}($nip)${spcb}...  ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
			${NCMD} "nodemgmt update q"
			echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]}  ...  ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
			${NCMD} "runup runup"
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
		echo
	done
	
	
	echo -e "${idsCL[Green]}All Nodes have been Updated.${idsCL[Default]}"
	echo
	echo
	end=`date +%s`
	runtime=$((end-start))
	echo "runtime: ${runtime}"
}

VCENTER-SSL(){
	VCHOSTNAME='vcenter.scity.us'
	#BASECERTDIR="/etc/letsencrypt/live"
	BASECERTDIR="/opt/nginx-proxy/ssl"
	CERTDIR="${BASECERTDIR}/${VCHOSTNAME}"
	SERVER="https://${VCHOSTNAME}"
	CRED="administrator@scity.vs:$(pass iDSVC)"	
	VCUSER="administrator@scity.vs"
	CURRENTLIVE=/etc/vmware-rhttpproxy/ssl/rui.crt
	
	CERT=/root/.acme.sh/$VCHOSTNAME/$VCHOSTNAME.cer
	KEY=/root/.acme.sh/$VCHOSTNAME/$VCHOSTNAME.key
	CHAIN=/root/.acme.sh/$VCHOSTNAME/fullchain.cer
	
	echo -en "${idsCL[LightCyan]}Checking days left on vCenter cert... ${idsCL[Default]}"
	VCCERTDAYS=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N)
	VCCERTDAYS=${VCCERTDAYS#*=}
	
	if [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" != "force" ]; then
		echo -e "${idsCL[Green]}${VCCERTDAYS} days left, Certificate is still valid, no noeed to update${idsCL[Default]}"
		echo
		exit 0
	elif [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" == "force" ]; then
		echo -e "${idsCL[Yellow]}${VCCERTDAYS} days left, forcing certificate update${idsCL[Default]}"
		echo
	else
		echo -e "${idsCL[Yellow]}${VCCERTDAYS} days left, Certificate needs to be updated${idsCL[Default]}"
		echo 
	fi
	
	LIVEMD5=$(ssh root@$VCHOSTNAME "md5sum $CURRENTLIVE | cut -d\  -f1")
	CURRENTMD5=$(md5sum $CERTDIR/cert.pem | cut -d\  -f1)
	if [ "$LIVEMD5" == "$CURRENTMD5" ] && [ "${1}" != "force" ]; then
		echo -e "${idsCL[Yellow]}Certificates remains the same, no newer certificates exist${idsCL[Default]}"
		echo
		exit 0
	fi
	
	echo -e "${idsCL[Green]}Updating certificates on vCenter... ${idsCL[Default]}"
	echo -e "${idsCL[LightCyan]}This process make take up to 10mins${idsCL[Default]}"
	echo
	
	# eval $(awk '{ print "export " $1 }' /etc/sysconfig/vmware-environment)

	scp $BASECERTDIR/$VCHOSTNAME/cert.pem root@${VCHOSTNAME}:$CERT
	scp $BASECERTDIR/$VCHOSTNAME/privkey.pem root@${VCHOSTNAME}:$KEY
	scp $BASECERTDIR/$VCHOSTNAME/fullchain.pem root@${VCHOSTNAME}:$CHAIN
	
	ssh root@$VCHOSTNAME "(printf '1\n%s\n' '$VCUSER'; sleep 1; printf '%s\n' '$(pass iDSVC)'; sleep 1; printf '2\n'; sleep 1; printf '%s\n%s\n%s\ny\n\n' '$CERT' '$KEY' '$CHAIN') | setsid /usr/lib/vmware-vmca/bin/certificate-manager"
	
	
	
	# SESSION_ID=$(curl -s -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' --header 'vmware-use-header-authn: test' --header 'vmware-api-session-id: null' -u $CRED "$SERVER/rest/com/vmware/cis/session" --insecure | python3 -c "import sys, json; print(json.load(sys.stdin)['value'])")
	# PRIVKEY=$(awk -v ORS='\\n' '1' "$CERTDIR/privkey.pem")
	# CERT=$(awk -v ORS='\\n' '1' "$CERTDIR/fullcert.pem")
	# REQUEST_BODY="{ \"spec\" : { \"cert\" : \"$CERT\", \"key\" : \"$PRIVKEY\"  } }"
	# curl --insecure -X PUT "$SERVER/rest/vcenter/certificate-management/vcenter/tls" \
	#                 -H "vmware-api-session-id: $SESSION_ID" \
	#                 -H "Content-type: application/json" \
	#                 -d "$REQUEST_BODY"
	# curl --insecure -X DELETE "$SERVER/rest/com/vmware/cis/session" -H "vmware-api-session-id: $SESSION_ID"
	
	# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
	# echo
	
	# read -n 1 -s -p "Press any key to continue"
	
	# echo -en "${idsCL[LightCyan]}Stopping vCenter services... ${idsCL[Default]}"
	# ssh -i ~/.ssh/id_rsa root@vcenter.scity.us 'service-control --stop --all' >/dev/null 2>&1
	# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
	#
	# echo -en "${idsCL[LightCyan]}Starting vCenter services... ${idsCL[Default]}"
	# ssh -i ~/.ssh/id_rsa root@vcenter.scity.us 'service-control --start --all' >/dev/null 2>&1
	# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
	
	echo
	echo -e "${idsCL[Green]}The vCenter certifcate has been updated${idsCL[Default]}"
	echo -e "${idsCL[LightCyan]}Don't forget to re-scan the vCenter connection in Veeam${idsCL[Default]}"
	echo
}
ADD_LOGROTATE_CRONTAB(){
	if ! crontab -l | grep -q "${FOLDER}/tmp-logrotate"; then
	  (crontab -l ; echo "0 */1 * * * logrotate -f ${FOLDER}/tmp-logrotate")| crontab -
	fi
	/bin/chmod 0644 ${FOLDER}/tmp-logrotate
	echo ""
	echo -e "${idsCL[Green]}The crontab entry has been made${idsCL[Default]}"
	echo ""
	read -n 1 -s -p "Press any key to continue"
	echo
}

UPDATE_DYNDNS(){
	if [ "${1}" != 'q' ]; then
		echo -e "${idsCL[LightCyan]}Updating Primary DNS A records with any WAN IP changes... ${idsCL[Default]}"
	fi
	
	OLDIP=`dig +short sc1.scity.us @ns1.scity.us`
	NEWIP=`dig +short myip.opendns.com @resolver1.opendns.com`
	
	if [ "${OLDIP}" != "${NEWIP}" ] && [ "${NEWIP}" != ";; connection timed out; no servers could be reached" ]; then
		dnss=0
		for i in {1..3}; do
		   curl -m 15 -s --head  --request GET https://wdns${i}.scity.us | grep "HTTP/2 302" > /dev/null && dnss=$i && break
		done
		
		if [ "${dnss}" != "0" ]; then
			/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=sc1.scity.us&myip=${NEWIP}" > /dev/null 2>&1
			/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=sc2.scity.us&myip=${NEWIP}" > /dev/null 2>&1
			/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=scity.us&myip=${NEWIP}" > /dev/null 2>&1
			/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=schroedercity.com&myip=${NEWIP}" > /dev/null 2>&1
			/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=dasrei.com&myip=${NEWIP}" > /dev/null 2>&1
			/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=snsinvest.com&myip=${NEWIP}" > /dev/null 2>&1
			/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=misfitsmc.com&myip=${NEWIP}" > /dev/null 2>&1
	
			SENDNOTICE "Updated WAN IP from '${OLDIP}' to '${NEWIP}'" "DYNDNS Updated"
		
			if [ "${1}" != 'q' ]; then
				echo -e "${idsCL[Green]}Updated WAN IP from '${OLDIP}' to '${NEWIP}'${idsCL[Default]}"
			fi
		else
			SENDNOTICE "Could not connect to any WDNS-PDNS server to update WAN IP" "DYNDNS ERROR"
		fi
	else
		if [ "${1}" != 'q' ]; then
			echo -e "${idsCL[Green]}WAP IP has not changed '${OLDIP}' = '${NEWIP}'${idsCL[Default]}"
		fi
	fi

	if [ "${1}" != 'q' ]; then
		echo ""
		read -n 1 -s -p "Press any key to continue"
		echo
	fi
}

RUN_COMMAND(){
	echo "${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}"
	for NTYPE in "${NODE_TYPES[@]}"; do
		echo -e "${idsST[Bold]}"; DIVIDER
		echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Nodes to run command { \`${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}\` }${idsCL[Default]}"
		DIVIDER; echo -e "${idsST[Reset]}"
		nid=1
		
		var=${NTYPE}_HOSTS[@]
		for nip in "${!var}"; do
			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
				else NCMD="ssh root@${nip}"; LH=''
			fi
			echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
			DIVIDER false green
			#echo "Running command: ${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}"
			${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}
			DIVIDER false green
			echo
			nid=`expr $nid + 1`
		done
		echo
	done
}

DOWNLIST(){
	if [ "${1}" = "report" ]; then
		unset idsCL idsBG idsST
		idsCL=('')
		idsBG=('')
		idsST=('')
	fi
	declare -i cw; declare -i spc1; declare -i c
	DOWN_LIST=`find ${FOLDER} -name "*.down"`
	
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
	fi
	echo 
	
	if [ "${DOWN_LIST}" == "" ]; then
		echo -e "${idsCL[LightCyan]}Nothing is down${idsCL[Default]}"
	else
		echo -e "${idsCL[LightGreen]}Current List of Items Down...${idsCL[Default]}"
		DIVIDER false yellow
		echo -e "Service/Host                            Host              Down_Time"
		DIVIDER false yellow
	
		cw=40; cw2=58
		for downitem in ${DOWN_LIST}; do
			ditem=${downitem##*/}
			ditem=${ditem//.down}
			host=${ditem%%-*}
			item=${ditem#*-}
		
			if [[ $ditem == *"-"* ]] && [ "${NM_SERVICES[${item}]}" != "" ]; then
				itemname=${NM_SERVICES[${ditem#*-}]}
			elif [[ $ditem == *"-"* ]] && [ "${NM_DOCKERS[${item}]}" != "" ]; then
				itemname="${NM_DOCKERS[${ditem#*-}]} (docker)"
			elif [[ $ditem == *"-"* ]] && [ "${REPL_DESC[${item}]}" != "" ]; then
				itemname="${REPL_DESC[${item}]} (replication)"
			else
				itemname='HOST DOWN';
			fi
		
			if [ -f ${FOLDER}/${ditem}.errtime ]; then
				toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${ditem}.errtime)
			else
				toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${ditem}.down)
			fi
		
			c=0; spc=''
			spc1=${cw}-${#itemname}
			#echo "HERE: $downitem ~ $ditem ~ $host ~ $item ~ $itemname ~ $spc1"
			until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
			c=0; spc2=''
			spc1=${cw2}-${#itemname}-${#host}-${spc1}
			until [ $c = ${spc1} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
			echo -e "${idsST[Bold]}${idsCL[Red]}${itemname}${spc}${host}${spc2}$(SHOW_TIME ${toterrtime})${idsST[Reset]}${idsCL[Default]}"
			DIVIDER false darkGray
		done
	fi
	echo 
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
		ENTER2CONTINUE
	fi
	echo ""
}

UPDATEPUSHOVERINFO(){
	echo
	if [ "${PUSHOVER_APP_TOKEN}" != "" ]; then
		echo -en "${idsCL[LightCyan]}Use current Pushover Info:\nApp Token: [${idsCL[White]}${PUSHOVER_APP_TOKEN}${idsCL[LightCyan]}\nUser Token: [${idsCL[White]}${PUSHOVER_USER_TOKEN}${idsCL[LightCyan]}] (Y/n)? ${idsCL[Default]}"
		read -n 1 choice
		case "$choice" in
			[Nn]) PUSHOVER_APP_TOKEN="";PUSHOVER_USER_TOKEN="";;esac
	fi
	if [ "${PUSHOVER_APP_TOKEN}" == "" ]; then
		echo; echo -e -n "${idsCL[LightCyan]}Pushover App Token: ${idsCL[Default]}"
		read -e PUSHOVER_APP_TOKEN
		echo; echo -e -n "${idsCL[LightCyan]}(optional) Pushover User Token: ${idsCL[Default]}"
		read -e PUSHOVER_USER_TOKEN
	fi
	
	declare -A host_ip
	declare -A host_name
	
	while read hostid hostname hostip hostlimits; do
		if [ "$hostid" != "id" ]; then
			host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
			host_name[${hostid}]=$hostname
		fi
	done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")

	for hostid in "${!host_ip[@]}"; do
		checkhost=$(CHECK_HOST ${host_ip[$hostid]})
		if [ "${checkhost}" != "false" ]; then
			echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Running commands${idsCL[Default]}${idsST[Reset]}"
			ssh root@${host_ip[$hostid]} "/bin/sed -i '/PUSHOVER_/d' /opt/idssys/nodemgmt/settings.conf"
			if [ "${PUSHOVER_APP_TOKEN}" != "" ]; then
				echo "putting in info 'root@${host_ip[$hostid]}'"
				ssh root@${host_ip[$hostid]} "echo PUSHOVER_APP_TOKEN=\\\"${PUSHOVER_APP_TOKEN}\\\" >> ${MMFOLDER}/settings.conf; echo 'yes1'"
			else
				echo "taking out info"
				ssh root@${host_ip[$hostid]} "echo \# PUSHOVER_APP_TOKEN=\\\"\\\" >> ${MMFOLDER}/settings.conf"
			fi
			if [ "${PUSHOVER_USER_TOKEN}" != "" ]; then
				ssh root@${host_ip[$hostid]} "echo PUSHOVER_USER_TOKEN=\\\"${PUSHOVER_USER_TOKEN}\\\" >> ${MMFOLDER}/settings.conf; echo 'yes2'"
			else 
				ssh root@${host_ip[$hostid]} "echo \# PUSHOVER_USER_TOKEN=\\\"\\\" >> ${MMFOLDER}/settings.conf"
			fi
			
		else
			echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
		fi
	done
	
}


GUI(){
	DISP_HEADER true false
	while :
	do
		echo " [1] Node Status"
		echo " [2] Node Services"
		echo ""
		echo " [3] New Site"
		echo " [4] Delete Site"
		echo ""
		echo " [5] New Certificate"
		echo " [6] Renew Certificates"
		echo " [7] List Certificates"
		echo ""
		echo " [8] Set folder permissions on nodes"
		echo " [9] Backup Node Settings and Files"
		echo ""
		echo " [D] Update Primary DNS Records"
		echo " [R] Add cron task to rotate wtmp & btmp log files"
		echo ""
		echo " [Q] Quit"
		echo ""
		echo ""
		echo -e -n "${idsCL[LightYellow]}Please select an [ActionItem] from above:${idsCL[Default]} "
		read -n 1 opt
		echo ""
		case $opt in
			1) STATUS; GUI;;
			2) SERVICES; GUI;;
			3) NEWSITE; GUI;;
			4) DELSITES; GUI;;
			5) NEWCERT; GUI;;
			6) CERTRENEW; GUI;;
			7) LISTCERTS; GUI;;
			8) SET-PERMISSIONS; GUI;;
			9) BACKUP; GUI;;
			[Dd]) UPDATE_DYNDNS; GUI;;
			[Rr]) ADD_LOGROTATE_CRONTAB; GUI;;
			[Qq]) EXIT1
				exit 0;;
			*)	echo "Thats an invaild option,";
				echo "please select a valid option only.";
				sleep 1;;
		esac
	done
}
if [ ${action-x} ]; then
	if [[ "${noheader}" != *" ${action} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [ "${2}" != "q" ] && [ "${1}" != "gui" ] && [ "${1}" != "nightlyrenew" ]; then
		DISP_HEADER
	fi
	case $action in
		newcert) NEWCERT ${2};;
		certrenew) CERTRENEW;;
		delssl) DEL-SSL ${2};;
		listcerts) LISTCERTS;;
		listcerts-npm) LISTCERTS_NPM;;
		copynpmcerts) COPYCERTS_NPM ${2};;
		checknpmcerts) CHECK_NPMCERTS;;
		checkcerts) CHECK-CERTS ${2};;
		nightlyrenew) NIGHTLYRENEW;;
		delsite) DELSITE ${2} ${3} ${4} ${5};;
		delsites) DELSITES;;
		newsite) NEWSITE ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} ${23} ${24} ${25};;
		update) ;;
		update-nodes) NODEUPDATE ${2};;
		update-dyndns) UPDATE_DYNDNS ${2};;
		backup) BACKUP;;
		downlist) 
			if [ "${2}" = "report" ]; then
				if [ "$(find ${FOLDER} -name '*.down')" != "" ]; then
					rm -f ${FOLDER}/status-downlist.lastrun
					DOWNLIST report >> ${FOLDER}/status-downlist.lastrun
					STATUSLOG="$(cat ${FOLDER}/status-downlist.lastrun)"
					SENDNOTICE "Downlist Report" "Downlist Report\n${STATUSLOG}"
				fi
			else
				DOWNLIST ${2}
			fi;;
		status)
			if [ -f "${FOLDER}/status-check.running" ]; then
				fd=`stat --format=%Y "${FOLDER}/status-check.running"`
				cbd=$((`date +%s` - 450))
				[ ${fd} -lt ${cbd} ] && rm -f "${FOLDER}/status-check.running"
			fi
			if [ ! -f "${FOLDER}/status-check.running" ]; then
				touch "${FOLDER}/status-check.running"
				if [ "${2}" == "report" ]; then
					rm -f ${FOLDER}/status-check.lastrun
					echo -e "$(date) - Starting status check #############################\n" > ${FOLDER}/status-check.lastrun
					STATUS report ${3} >> ${FOLDER}/status-check.lastrun
					echo -e "\n\n$(date) - Finished status check #############################\n" >> ${FOLDER}/status-check.lastrun
					if [ "${3}" = "email" ]; then
						echo -e "\n\n" >> ${FOLDER}/status-check.lastrun
						DOWNLIST report >> ${FOLDER}/status-check.lastrun
						STATUSLOG="$(cat ${FOLDER}/status-check.lastrun)"
						SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}"
					fi
				else
					STATUS ${2} ${3}
				fi
				rm -f "${FOLDER}/status-check.running"
			else
				echo -e "${idsCL[LightYellow]}Status check already running, please wait and try again${idsCL[Default]}"
				echo
			fi;;
		checkdocker) CHECK_DOCKER_SERVICES ${2};;
		status-check) STATUS-CHECK ${2};;
		set-permissions) SET-PERMISSIONS ${2};;
		service)
			if [[ ( "$2" = "giteax" ) || ( "$2" = "powerdns-admin" ) ]] && [ "$3" = "start" ]; then
				#SET-PERMISSIONS ${2}
				SERVICE ${2} start
			elif [[ ( "$2" = "giteax" ) || ( "$2" = "powerdns-admin" ) ]] && [ "$3" = "restart" ]; then
				SERVICE ${2} stop
				#SET-PERMISSIONS ${2}
				SERVICE ${2} start
			elif [ "$2" = "web" ]; then
				SERVICE php5.6-fpm ${3} ${4}
				SERVICE php7.2-fpm ${3} ${4}
				SERVICE php7.3-fpm ${3} ${4}
				SERVICE php7.4-fpm ${3} ${4}
				SERVICE php8.2-fpm ${3} ${4}
				SERVICE nginx ${3} ${4}
			elif [ "$2" = "glusterdxx" ]; then
				SERVICE glusterd stop
			else
				SERVICE ${2} ${3} ${4}
			fi;;
		services) SERVICES;;
		singleservercheck) SINGLE_SERVER_SERVICE_CHECK ${2} ${3};;
		haproxy-config) HAPROXY-CONFIG;;
		keepalive-config) KEEPALIVE-CONFIG;;
		vc-ssl) VCENTER-SSL ${2};;
		run) RUN_COMMAND ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15};;
		update-pushover) UPDATEPUSHOVERINFO;;
		gui) GUI;;
		*)
			echo -e "Usage: ${idsCL[LightCyan]}nodemgmt${idsCL[Default]} {"
			echo -e "	  ${idsCL[Yellow]}backup${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}certrenew${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}checkcerts${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}delsite${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}delsites${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}delssl${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}gui${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}haproxy-config${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}listcerts${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}newcert${idsCL[Default]} [domain(,s)]"
			echo -e "	  ${idsCL[Yellow]}newsite${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}nightlyrenew${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}run${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}service${idsCL[Default]} [service] [action]"
			echo -e "	  ${idsCL[Yellow]}services${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}set-permissions${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}status${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}status-check${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}update-nodes${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}update-dyndns${idsCL[Default]}"
			echo -e "	  ${idsCL[Yellow]}vc-ssl${idsCL[Default]}"
			echo -e "}"
			echo ""
			echo ""
			exit 0;;
	esac
else
	GUI
fi

exit 0