#!/usr/bin/env bash

STATUS(){
	start=`date +%s`
	ST_ACTION=${1}
	if [ "${ST_ACTION}" = "report" ]; then
		unset idsCL idsBG idsST
		idsCL=('')
		idsBG=('')
		idsST=('')
	elif [ "${ST_ACTION}" = "sync" ]; then
		ST_ACTION=repl
	fi
	declare -i cw; declare -i spc1; declare -i c
	
	########################
	if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
		if [ "${ST_ACTION}" != "check" ]; then
			echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}"
		fi
		for NTYPE in "${NODE_TYPES[@]}"; do
			PH=${NTYPE}_HOSTS[0]
			if [ ! -f ${FOLDER}/${!PH}.down ]; then
				PH_CMD="ssh root@${!PH}"
				var=${NTYPE}_REPL_CHECK[@]
				if [ ! -z ${!var+x} ]; then
					for rcheck in "${!var}"; do
						# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
						${PH_CMD} rm -f ${FOLDER}/test.repl
						daterun=`date +%Y-%m-%d-%H-%M-%S`
						${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" &
					done
				fi
			fi
		done
		# replstart=`date +%s`
		if [ "${ST_ACTION}" != "check" ]; then
			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
			echo
		fi
	fi

	########################
	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then
		if [ ! -z ${LOCAL_SERVICES+x} ]; then
			if [ "${ST_ACTION}" != "check" ]; then
				lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
				uptime=`uptime -p`
				echo -e "${idsST[Bold]}"; DIVIDER
				echo -e "${idsCL[Yellow]} ${NODE_HOSTNAME} Service Status -localhost-${idsCL[Default]}"
				DIVIDER; echo -e "${idsST[Reset]}"
				echo -e " ${idsCL[LightCyan]}${uptime}${idsCL[Default]}"
			fi
			# if [ "${ST_ACTION}" != "check" ]; then
			# 	lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
			# 	uptime=`uptime -p`
			# 	echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
			# 	DIVIDER false green
			# fi
			for srvc in "${LOCAL_SERVICES[@]}"; do
				if [ "${ST_ACTION}" != "check" ]; then
					if [ "${ST_ACTION}" != "report" ]; then
						c=0; cw=30; spc=''
						spc1=${cw}-${#NM_SERVICES[${srvc}]}
						until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
					else
						spc=' '
					fi
					echo -en " ${NM_SERVICES[${srvc}]}${spc}: "
				fi
				
				if [ "$(systemctl is-active ${srvc})" != "active" ]; then
					if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down ]; then
						if [ "${ST_ACTION}" != "check" ]; then
							echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
						fi
						SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1
						touch ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down
						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
					else
						errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down)
						if [ $errtime -gt ${RENOTIFY} ]; then
							if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime ]; then
								mv ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime
							fi
							toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime)
							if [ "${ST_ACTION}" != "check" ]; then
								echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
							fi
							touch ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down
						fi
					fi
				else
					if [ "${ST_ACTION}" != "check" ]; then
						echo -e "${idsCL[Green]}Running${idsCL[Default]}"
					fi
					if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down ]; then
						if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime ]; then
							toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime)
						else
							toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down)
						fi
						rm -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.*
						SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE}
						
					fi
				fi
			done
			
			if [ "${ST_ACTION}" != "check" ]; then echo; fi
		fi
		########################
		
		for NTYPE in "${NODE_TYPES[@]}"; do
			nid=1
			
			dockers=${NTYPE}_DOCKER[@]
			hosts=${NTYPE}_HOSTS[@]
			
			var=${NTYPE}_HOSTS[@]
			
			if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" == "dockers" ]; then
				GOFORCHECK=false;
			else
				GOFORCHECK=true;
			fi
				
			if [ ${GOFORCHECK} = true ]; then
			
				if [ "${ST_ACTION}" != "check" ]; then
					echo -e "${idsST[Bold]}"; DIVIDER
					if [ "${ST_ACTION}" = "dockers" ]; then
						echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
					else
						echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
					fi
					DIVIDER; echo -e "${idsST[Reset]}"
				fi
				for nip in "${!var}"; do
					[ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}"
					
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
						else NCMD="ssh root@${nip}"; LH=''
					fi
					if [ "${NCMD}" != "" ]; then
						checkhost=$(CHECK_HOST ${nip})
					fi
					if [ "${checkhost}" != "false" ]; then
						if [ "${ST_ACTION}" != "check" ]; then
							#echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}"
							if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then
								uptime=`${NCMD} uptime -p`
								echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
							else
								echo -e "${idsCL[Default]}"
							fi
							
							DIVIDER false green
						fi
						if [ -f ${TMPFOLDER}/${nip}.down ]; then
							if [ -f ${TMPFOLDER}/${nip}.errtime ]; then
								toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
							else
								toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
							fi
							rm -f ${TMPFOLDER}/${nip}.*
							SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
						fi
						
						if [ "${ST_ACTION}" != "dockers" ]; then
							srvcs=${NTYPE}_SERVICES_CHECK[@];
							srvcstotest="$(join_by " " ${!srvcs})"
							srvctst=(`${NCMD} systemctl is-active ${srvcstotest}`)
							sr=0
							for srvc in "${!srvcs}"; do
								[ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false || NOGOCHK=true
								if [ ${NOGOCHK} == true ]; then
									if [ "${ST_ACTION}" != "check" ]; then
										if [ "${ST_ACTION}" != "report" ]; then
											c=0; cw=30; spc=''; spc1=${cw}-${#NM_SERVICES[${srvc}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
										else
											spc=' '
										fi
										echo -en " ${NM_SERVICES[${srvc}]}$spc: "
									fi
									if [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" == "mysql" ]; then
										[ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false
									elif [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
										[ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false
									elif [ "${srvctst[$sr]}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then
										mysqlgo=true
									fi
									# echo "mysqlgo=${mysqlgo}"
									if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
										if [ ! -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then
											[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
											touch ${TMPFOLDER}/${nip}.${srvc}.down
											# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
											echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
										else
											errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down)
											if [ $errtime -gt ${RENOTIFY} ]; then
												[ ! -f ${TMPFOLDER}/${nip}.${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}.${srvc}.down ${TMPFOLDER}/${nip}.${srvc}.errtime
												toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.errtime)
												touch ${TMPFOLDER}/${nip}.${srvc}.down
												SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
												# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
											
											elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ]; then
												SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1

											fi
											[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
										fi
									else
										[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
										if [ -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then
											[ -f ${TMPFOLDER}/${nip}.${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down)
											SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
											echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}

											rm -f ${TMPFOLDER}/${nip}.${srvc}.*

										fi
										# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
										if [ "${srvc}" == "gitea" ]; then
											rm -f ${FOLDER}/*.${srvc}.*
										fi
									fi
									[ "${mysqlgo}" = "true" ] && unset mysqlgo
								fi
								sr=`expr $sr + 1`
							done
							
							
							
							
							
							# for srvc in "${!var2}"; do
							# 	NOGOCHK=true;
							# 	[ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false;
							# 	if [ ${NOGOCHK} = true ]; then
							# 		if [ "${ST_ACTION}" != "check" ]; then
							# 			if [ "${ST_ACTION}" != "report" ]; then
							# 				c=0; cw=30; spc=''
							# 				spc1=${cw}-${#NM_SERVICES[${srvc}]}
							# 				until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
							# 			else
							# 				spc=' '
							# 			fi
							# 			echo -en " ${NM_SERVICES[${srvc}]}$spc: "
							# 		fi
							# 		srvctst=$(${NCMD} systemctl is-active ${srvc})
							# 		if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then
							# 			mysqlgo=false;
							# 			[ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true;
							# 		elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
							# 			mysqlgo=false;
							# 			[ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true;
							# 		elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then
							# 			mysqlgo=true
							# 		fi
							# 		# echo "mysqlgo=${mysqlgo}"
							# 		if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
							# 			if [ ! -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then
							# 					if [ "${ST_ACTION}" != "check" ]; then
							# 						echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
							# 					fi
							# 					SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
							# 					touch ${TMPFOLDER}/${nip}.${srvc}.down
							# 					echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
							#
							# 			else
							# 				errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down)
							# 				if [ $errtime -gt ${RENOTIFY} ]; then
							# 					if [ ! -f ${TMPFOLDER}/${nip}.${srvc}.errtime ]; then
							# 						mv ${TMPFOLDER}/${nip}.${srvc}.down ${TMPFOLDER}/${nip}.${srvc}.errtime
							# 					fi
							# 					toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.errtime)
							# 					touch ${TMPFOLDER}/${nip}.${srvc}.down
							# 					if [ "${srvc}" = "mysql" ]; then
							# 						${NCMD} systemctl restart ${srvc}
							# 					fi
							# 				fi
							# 				if [ "${ST_ACTION}" != "check" ]; then
							# 					echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
							# 				fi
							# 			fi
							# 		else
							# 			if [ "${ST_ACTION}" != "check" ]; then
							# 				echo -e "${idsCL[Green]}Running${idsCL[Default]}"
							# 			fi
							# 			if [ -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then
							# 				if [ -f ${TMPFOLDER}/${nip}.${srvc}.errtime ]; then
							# 					toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.errtime)
							# 				else
							# 					toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down)
							# 				fi
							# 				SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
							# 				echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
							#
							# 				rm -f ${TMPFOLDER}/${nip}.${srvc}.down
							# 				rm -f ${TMPFOLDER}/${nip}.${srvc}.errtime
							#
							# 			fi
							# 			# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
							# 			if [ "${srvc}" == "gitea" ]; then
							# 				rm -f ${FOLDER}/*.${srvc}.down
							# 				rm -f ${FOLDER}/*.${srvc}.errtime
							# 			fi
							# 		fi
							# 		[ "${mysqlgo}" = "true" ] && unset mysqlgo
							# 	fi
							# done
						fi
						
						if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then
							if [ "${ST_ACTION}" != "check" ]; then
								echo
								echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
								echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}"
							fi
						
							for docker in "${!dockers}"; do
								if [ "${ST_ACTION}" != "check" ]; then
									if [ "${ST_ACTION}" != "report" ]; then
										c=0; cw=30; spc=''
										spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
										until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
									else
										spc=' '
									fi
									echo -en " ${NM_DOCKERS[${docker}]}$spc: "
								fi
								
								if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
									if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
										${NCMD} docker rm ${docker} >/dev/null 2>&1
									fi
									${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
									sleep 10s
									if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
										if [ "${ST_ACTION}" != "check" ]; then
											echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
										fi
										SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed"
										rm -f ${TMPFOLDER}/${nip}-${docker}.down
										rm -f ${TMPFOLDER}/${nip}-${docker}.errtime
										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE}

									else
										if [ "${ST_ACTION}" != "check" ]; then
											echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
										
											if [ ! -f ${TMPFOLDER}/${nip}-${docker}.down ]; then
												SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1
												touch ${TMPFOLDER}/${nip}-${docker}.down
												echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE}
											else
												errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${docker}.down)
												if [ $errtime -gt ${RENOTIFY} ]; then
													if [ ! -f ${TMPFOLDER}/${nip}-${docker}.errtime ]; then
														mv ${TMPFOLDER}/${nip}-${docker}.down ${TMPFOLDER}/${nip}-${docker}.errtime
													fi
													toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${docker}.errtime)
													if [ "${ST_ACTION}" != "check" ]; then
														echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
													fi
													touch ${TMPFOLDER}/${nip}-${docker}.down
												fi
											fi
										
										fi
									fi
								else
									if [ "${ST_ACTION}" != "check" ]; then
										echo -e "${idsCL[Green]}Running${idsCL[Default]}"
									fi
									rm -f ${TMPFOLDER}/${nip}-${docker}.down
									rm -f ${TMPFOLDER}/${nip}-${docker}.errtime
								fi
							done
						fi
					
					else
						if [ ! -f ${TMPFOLDER}/${nip}.down ]; then
							touch ${TMPFOLDER}/${nip}.down
							if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
								touch ${TMPFOLDER}/${nip}.errtime
							fi
							if [ "${ST_ACTION}" != "check" ]; then
								toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
								echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
							fi
							# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1
							echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE}
						else
							if [ "${ST_ACTION}" != "check" ]; then
								toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
								echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
							fi
							errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
							if [ $errtime -gt 60 ] && [ $errtime -lt 180 ]; then
								SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
							fi
							# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then
							# 	echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}
							# fi
						fi

					fi
					if [ "${ST_ACTION}" != "check" ]; then echo; fi
					nid=`expr $nid + 1`
				done

			fi #GOFORCHECK
		
		done
		
	fi
	########################
	## REPLICATION CHECK
	########################
	
	if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
		if [ "${ST_ACTION}" != "check" ]; then
			echo -e "${idsST[Bold]}"; DIVIDER
			echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
			DIVIDER; echo -e "${idsST[Reset]}"
			echo -en " ${idsCL[LightCyan]}Starting processes to collect/monitor replication status : "
		fi
		for NTYPE in "${NODE_TYPES[@]}"; do
			PH=${NTYPE}_HOSTS[0]
			PH_CMD="ssh root@${!PH}"
			var=${NTYPE}_REPL_CHECK[@]
			if [ ! -z ${!var+x} ]; then
				var=${NTYPE}_HOSTS[@]
				for nip in "${!var}"; do
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
						if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
							var2=${NTYPE}_REPL_CHECK[@]
							for rcheck in "${!var2}"; do
								REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
							done
						fi
					fi
				done
			fi
		done
		[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"	
		
		for NTYPE in "${NODE_TYPES[@]}"; do
			PH=${NTYPE}_HOSTS[0]
			PH_CMD="ssh root@${!PH}"
			var=${NTYPE}_REPL_CHECK[@]
			if [ ! -z ${!var+x} ]; then
				var=${NTYPE}_HOSTS[@]
				for nip in "${!var}"; do
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
						if [ "${ST_ACTION}" != "check" ]; then
							echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
							DIVIDER false green
						fi
						if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
							var2=${NTYPE}_REPL_CHECK[@]
							for rcheck in "${!var2}"; do
								if [ "${ST_ACTION}" != "check" ]; then
									if [ "${ST_ACTION}" != "report" ]; then
										c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
									else
										spc=' '
									fi
									echo -en " ${REPL_DESC[${rcheck}]}${spc}: "

								fi
								
								checked=""
								until [ "${checked}" != "" ]; do
									if [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
										checked=good
									elif [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
										checked=timeout
									fi
									rm -f ${TMPFOLDER}/repl.${rcheck}.${nip}.*
								done

								if [ "${checked}" == "timeout" ]; then
									[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
									if [ ! -f ${TMPFOLDER}/${nip}-${rcheck}.down ]; then
										touch ${TMPFOLDER}/${nip}-${rcheck}.down
										SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
									fi
								else
									[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
									if [ -f ${TMPFOLDER}/${nip}-${rcheck}.down ]; then
										rm -f ${TMPFOLDER}/${nip}-${rcheck}.down
										SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
									fi
								fi				
							done
						else
							if [ "${ST_ACTION}" != "check" ]; then
								[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}"
								[ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}"
							fi
						fi
						if [ "${ST_ACTION}" != "check" ]; then echo; fi
					fi
				done
			fi
		done
		##########################
		# REMOVE REPL CHECK FILES
		##########################
		if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
			if [ "${ST_ACTION}" != "check" ]; then
				echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
			fi
			#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
			#LBHCMD="ssh root@${LB_HOSTS[0]}"
			#WHCMD="ssh root@${WEB_HOSTS[0]}"
			for NTYPE in "${NODE_TYPES[@]}"; do
				PH=${NTYPE}_HOSTS[0]
				if [ ! -f ${FOLDER}/${!PH}.down ]; then
					PH_CMD="ssh root@${!PH}"
					var=${NTYPE}_REPL_CHECK[@]
					if [ ! -z ${!var+x} ]; then
						for rcheck in "${!var}"; do
							${PH_CMD} rm -f ${FOLDER}/test.repl
							daterun=`date +%Y-%m-%d-%H-%M-%S`
							if [ "${PH_CMD}" = "" ]; then
								rm -f ${FOLDER}/test.repl &
								rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
							else
								${PH_CMD} rm -f ${FOLDER}/test.repl &
								${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
							fi
						done
					fi
				fi
			done
			rm -Rf ${FOLDER}/test.repl
			if [ "${ST_ACTION}" != "check" ]; then
				echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
				echo
			fi
		fi	
	fi
	
	########################
	## FREE SPACE CHECK
	########################
	#if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then
	if [ "${ST_ACTION}" = "freespace" ]; then
		if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then
			for NTYPE in "${NODE_TYPES[@]}"; do
				if [ "${ST_ACTION}" != "check" ]; then
					echo -e "${idsST[Bold]}"; DIVIDER
					echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Free Space Scan${idsCL[Default]}"
					DIVIDER; echo -e "${idsST[Reset]}"
				fi
				nid=1
				var=${NTYPE}_HOSTS[@]
				for nip in "${!var}"; do
					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
						else NCMD="ssh root@${nip}"; LH=''
					fi
					if [ "${ST_ACTION}" != "check" ]; then
						echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
						DIVIDER false green
					fi
					
					[ "${ST_ACTION}" != "check" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
					declare -A partitions
					DRIVEINFO=$(ssh root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
					DRIVEINFO=(${DRIVEINFO})
					echo -en "\e[1A";
					echo -e "\e[0K\r"
					
					NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
					for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
						ii=$((${i}*3))
						pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
						pname=${pname#*vg-}
						if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then
							freespace=${DRIVEINFO[$((${ii}+2))]//M/}
							if [ "${ST_ACTION}" != "check" ]; then
								c=0; cw=20; spc=''
								spc1=${cw}-${#pname}
								until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
								echo -en " ${pname}$spc: "
							fi
							if [[ ${freespace} -gt 1024 ]]; then
								fsgb=$(bc <<< "scale=2; ${freespace}/1024")
								fsdsp="${fsgb} GB"
							else
								fsdsp="${freespace} MB"
							fi
							if [ "${freespace}" -le "1024" ]; then
								fs_status='error'
								fs_status_color='Red'
								SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free" 1
							elif [ "${freespace}" -le "5120" ]; then
								fs_status='warn'
								fs_status_color='Yellow'
								SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free"
							else
								fs_status=''
								fs_status_color='Green'
							fi
							[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
						fi
					done
					
					# for partition in "${partitions3[@]}"; do
					# 	if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then
					# 		pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
					# 		pname=${pname#*vg-}
					# 		if [ "${ST_ACTION}" != "check" ]; then
					# 			c=0; cw=20; spc=''
					# 			spc1=${cw}-${#pname}
					# 			until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
					# 			echo -en " ${pname}$spc: "
					# 		fi
					#
					# 		# [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition}
					# 		# freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1
					#
					# 		if [ "${freespace}" -le "1000" ]; then
					# 			fs_status='error'
					# 			fs_status_color='Red'
					# 			SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" 1
					#
					# 		elif [ "${freespace}" -le "5000" ]; then
					# 			fs_status='warn'
					# 			fs_status_color='Yellow'
					# 			SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free"
					#
					# 		else
					# 			fs_status=''
					# 			fs_status_color='Green'
					#
					# 		fi
					# 		if [ "${ST_ACTION}" != "check" ]; then
					# 			if [[ ${freespace} -gt 1000 ]]; then
					# 				fsgb=$(bc <<< "scale=2; ${freespace}/1000")
					# 				fsdsp="${fsgb} GB"
					# 			else
					# 				fsdsp="${freespace} MB"
					# 			fi
					# 			echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
					#
					# 		fi
					# 	fi
					# done

					nid=`expr $nid + 1`
					echo
				done
			done
		fi
	fi
	########################
	########################
	
	if [ "${ST_ACTION}" != "check" ]; then
		echo ""
		if [ -z $action ] || [ "${action}" = "gui" ]; then
			DIVIDER true
			ENTER2CONTINUE
		fi
	fi
	end=`date +%s`
	runtime=$((end-start))
	echo "runtime: ${runtime}"
}

REPLCHECK(){
	rcheck=${1}
	nip=${2}
	PH_CMD=${3}
	if [ "${4}" != "check" ]; then
		timeout=`date --date='1 minutes' +%s`
	else
		timeout=`date --date='2 minutes' +%s`
	fi
	checked=false
	until [ "${checked}" == "" ]; do
		if [ "${PH_CMD}" == "" ]; then
			ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
		else
			ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
		fi
		[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
	done
	[ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good
}