#!/usr/bin/env bash
STATUS(){
	start=$(date +%s)
	log_start=$(date "+%Y-%m-%d %H:%M:%S")
	
	if [ "${1}" != "" ] && [ "${NM_NODETYPES[${1^^}]}" != "" ]; then
		ntss=${1^^}
		ntypesel=(${ntss})
		STATUS_ACTION=${2}
	else
		ntss=""
		ntypesel=(${NM_NODE_TYPES[@]})
		[ "${1}" == "sync" ] && STATUS_ACTION=repl || STATUS_ACTION=${1}
	fi
	# while [ $# -gt 0 ]; do
	#     case "$1" in
	#         -site) DEL_SITE=${2};;
	#         -ssl) DEL_SSL=${2};;
	#
	#     esac
	# 	shift
	# done
	
	PRI_CW=40
	DV_LEN=80
	declare -i cw; declare -i spc1; declare -i c
	

	
	############################
	## REPLICATION CHECK SETUP
	############################
	
	([ ! -f ${NM_TMPFOLDER}/.replcheck ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replcheck)) -ge 300 ]) && REPLRUN=1 || REPLRUN=0
	# if [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "" ]; then
	if [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "" ] || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && ([ "${STATUS_ACTION}" == "services" ] || [ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "repl" ]) && [ "${NM_REPL_CHECK[${ntss}]}" != "" ]); then
		if [ "${STATUS_ACTION}" != "check" ]; then
			echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}"
		fi
		for NTYPE in "${ntypesel[@]}"; do
			REPLSTART ${NTYPE} &
		done
		if [ "${STATUS_ACTION}" != "check" ]; then
			echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"
			echo -en "${idsCL[LightCyan]}Starting processes to collect/monitor replication check : "
		fi
		for NTYPE in "${ntypesel[@]}"; do
			PH=${NTYPE}_HOSTS[0]
			PH_CMD="ssh root@${!PH}"
			var=${NTYPE}_REPL_CHECK[@]
			if [ ! -z ${!var+x} ]; then
				var=${NTYPE}_HOSTS[@]
				for nip in "${!var}"; do
					if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
						if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
							var2=${NTYPE}_REPL_CHECK[@]
							for rcheck in "${!var2}"; do
								REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${STATUS_ACTION}" & >/dev/null 2>&1
							done
						fi
					fi
				done
			fi
		done
		[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"
	fi
	if [ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "services" ] || [ "${STATUS_ACTION}" == "dockers" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "all" ]; then
		
		[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]}Starting processes to collect node service & docker info : "
		for NTYPE in "${ntypesel[@]}"; do
			STATUS_NODE ${NTYPE} > ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.running 2>/dev/null &
		done
		[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"			
		
		########################
		## LOCALHOST CHECK
		########################
				
		if [ ! -z ${LOCAL_SERVICES+x} ] && [ "${STATUS_ACTION}" != "dockers" ] && [ "${STATUS_ACTION}" != "services" ] && [ "${ntss}" == "" ]; then
			cpu_usage=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
			
			if [ "${STATUS_ACTION}" != "check" ]; then
				lip=${RUN_NODE_IP}
				echo -e "\n$(DIVIDER . . ${DV_LEN})\n${idsCL[Yellow]}${idsST[Bold]} LOCALHOST Service Status${idsST[Reset]}${idsCL[Default]}"
				echo -e "$(DIVIDER . . ${DV_LEN})"
				echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} ${idsST[Reset]}${idsCL[LightCyan]}[${lip}]"
				if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then
					uptime=`uptime -p`
					echo -e "${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
				else
					echo -e "${idsCL[Default]}"
				fi
				DIVIDER . green `expr ${DV_LEN} - 10`
				
			fi
			# if [ "${STATUS_ACTION}" != "check" ]; then
			# 	lip=${RUN_NODE_IP}
			# 	uptime=`uptime -p`
			# 	echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
			# 	DIVIDER false green ${DV_LEN}
			# fi
			for srvc in "${LOCAL_SERVICES[@]}"; do
				if [ "${STATUS_ACTION}" != "check" ]; then
					if [ "${STATUS_ACTION}" != "report" ]; then
						c=0; cw=${PRI_CW}; spc=''
						spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
						until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
					else
						spc=' '
					fi
					echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
				fi
				
				if [ "$(systemctl is-active ${srvc})" != "active" ]; then
					if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
						[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
						SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
						touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
						echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
					else
						errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down))
						if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
							[ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
							toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
							[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
							touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
						fi
					fi
				else
					[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}"
					if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
						if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
							toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
						else
							toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down))
						fi
						rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
						SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
						echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE}
						
					fi
				fi
			done
		fi
		
		###################################
		## NODE SERVICE AND DOCKER CHECK
		###################################
		
		nc_count=0; completed=false
		until [ "${completed}" == "true" ]; do
			for nodestatus in ${STATUSRUN_TMPFOLDER}/status-check.*.done; do
				NTS=$(grep -oP '(?<=status-check.).*?(?=.done)' <<< "${nodestatus}")
				if [ "${NTS}" != "*" ]; then
					nodestatus=$(cat ${STATUSRUN_TMPFOLDER}/status-check.${NTS}.running)
					[ "${nodestatus}" != "" ] && echo -e "${nodestatus}"
					rm -f ${STATUSRUN_TMPFOLDER}/status-check.${NTS}.done
					((nc_count++))
					# echo "${NTS} == ${nc_count} == ${nodestatus}"
				fi
			done
			[ ${nc_count} -eq ${#ntypesel[@]} ] && completed=true
		done
		
		
		# for NTYPE in "${ntypesel[@]}"; do
		# 	completed=false
		# 	until [ "${completed}" == "true" ]; do
		# 		if [ -f ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.done ]; then
		# 			completed=true
		# 			echo -e "$(cat ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.running)"
		# 		fi
		# 	done
		# done
		
	fi
	
	########################
	## REPLICATION CHECK
	########################
	
	# if [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "" ]; then
	if [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "" ] || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]); then
		if [ "${STATUS_ACTION}" != "check" ]; then
			echo -e "${idsST[Bold]}"; DIVIDER . . ${DV_LEN}
			echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
			DIVIDER . . ${DV_LEN}; echo -en "${idsST[Reset]}"
		fi
		
		for NTYPE in "${ntypesel[@]}"; do
			PH=${NTYPE}_HOSTS[0]
			PH_CMD="ssh root@${!PH}"
			var=${NTYPE}_REPL_CHECK[@]
			if [ ! -z ${!var+x} ]; then
				var=${NTYPE}_HOSTS[@]
				for nip in "${!var}"; do
					if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
						if [ "${STATUS_ACTION}" != "check" ]; then
							echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]}[${!PH}] <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
							DIVIDER false green `expr ${DV_LEN} - 10`
						fi
						if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
							var2=${NTYPE}_REPL_CHECK[@]
							for rcheck in "${!var2}"; do
								if [ "${STATUS_ACTION}" != "check" ]; then
									if [ "${STATUS_ACTION}" != "report" ]; then
										c=0; cw=${PRI_CW}; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
									else
										spc=' '
									fi
									echo -en " ${idsCL[White]}${NM_REPL_DESC[${rcheck}]}${spc}${idsCL[Default]}: "

								fi
								
								checked=""
								until [ "${checked}" != "" ]; do
									if [ -f ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
										checked=good
									elif [ -f ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
										checked=timeout
									fi
								done

								if [ "${checked}" == "timeout" ]; then
									[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
									if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
										touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
									else
										if ([ -f ${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)) -gt 60 ]); then
											[ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime
											echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
											SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
It has been down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)))" 1
											touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent
											touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
										fi
									fi
										
								elif [ "${checked}" == "good" ]; then
									[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}"
									if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
										if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ]; then
											SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)))"
											echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}
										fi
										rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl*
									fi
									ssh -q root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;'
								fi			
							done
														
						else
							if [ "${STATUS_ACTION}" != "check" ]; then
								[ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
								[ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
							fi
						fi
						[ "${STATUS_ACTION}" != "check" ] && echo
						
					fi
										
				done
			fi
		done
		##########################
		# REMOVE REPL CHECK FILES
		##########################
		[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}"
		for NTYPE in "${ntypesel[@]}"; do
			PH=${NTYPE}_HOSTS[0]
			if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
				PH_CMD="ssh root@${!PH}"
				var=${NTYPE}_REPL_CHECK[@]
				if [ ! -z ${!var+x} ]; then
					for rcheck in "${!var}"; do
						if [ "${PH_CMD}" == "" ]; then
							rm -f ${NM_FOLDER}/test.${STATUS_START//-/}.repl &
							rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.${STATUS_START//-/}.repl &
						else
							${PH_CMD} rm -f ${NM_FOLDER}/test.${STATUS_START//-/}.repl &
							${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.${STATUS_START//-/}.repl &
						fi
					done
				fi
			fi
		done
		rm -f ${NM_FOLDER}/test.${STATUS_START//-/}.repl
		touch ${NM_TMPFOLDER}/.replcheck
		[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}\n"
	fi
	
	########################
	## FREE SPACE CHECK
	########################
	#if [ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "freespace" ]; then
	if [ "${STATUS_ACTION}" == "freespace" ]; then
		for NTYPE in "${ntypesel[@]}"; do
			if [ "${STATUS_ACTION}" != "check" ]; then
				echo -en "${idsST[Bold]}"; DIVIDER . . ${DV_LEN}
				echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node(s) Free Space Scan${idsCL[Default]}"
				DIVIDER . . ${DV_LEN}; echo -e "${idsST[Reset]}"
			fi
			var=${NTYPE}_HOSTS[@]
			for nip in "${!var}"; do
				if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
					else NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH=''
				fi
				if [ "${STATUS_ACTION}" != "check" ]; then
					echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
					DIVIDER false green `expr ${DV_LEN} - 10`
				fi
				
				[ "${STATUS_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
				declare -A partitions
				DRIVEINFO=$(ssh -o ConnectTimeout=3 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
				DRIVEINFO=(${DRIVEINFO})
				if  [ "${2}" != "report" ]; then
					echo -en "\e[1A"
					echo -e "\e[0K\r"
				fi
				
				NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
				for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
					ii=$((${i}*3))
					pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
					pname=${pname#*vg-}
					if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then
						freespace=${DRIVEINFO[$((${ii}+2))]//M/}
						if [ "${STATUS_ACTION}" != "check" ]; then
							c=0; cw=20; spc=''
							spc1=${cw}-${#pname}
							until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
							echo -en " ${pname}$spc: "
						fi
						if [[ ${freespace} -gt 1024 ]]; then
							fsgb=$(bc <<< "scale=2; ${freespace}/1024")
							fsdsp="${fsgb} GB"
						else
							fsdsp="${freespace} MB"
						fi
						if [ "${freespace}" -le "1024" ]; then
							fs_status='error'
							fs_status_color='Red'
							SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free" 1
						elif [ "${freespace}" -le "5120" ]; then
							fs_status='warn'
							fs_status_color='Yellow'
							SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free"
						else
							fs_status=''
							fs_status_color='Green'
						fi
						[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
					fi
				done
				
				# for partition in "${partitions3[@]}"; do
				# 	if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then
				# 		pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
				# 		pname=${pname#*vg-}
				# 		if [ "${STATUS_ACTION}" != "check" ]; then
				# 			c=0; cw=20; spc=''
				# 			spc1=${cw}-${#pname}
				# 			until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
				# 			echo -en " ${pname}$spc: "
				# 		fi
				#
				# 		# [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition}
				# 		# freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1
				#
				# 		if [ "${freespace}" -le "1000" ]; then
				# 			fs_status='error'
				# 			fs_status_color='Red'
				# 			SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free" 1
				#
				# 		elif [ "${freespace}" -le "5000" ]; then
				# 			fs_status='warn'
				# 			fs_status_color='Yellow'
				# 			SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free"
				#
				# 		else
				# 			fs_status=''
				# 			fs_status_color='Green'
				#
				# 		fi
				# 		if [ "${STATUS_ACTION}" != "check" ]; then
				# 			if [[ ${freespace} -gt 1000 ]]; then
				# 				fsgb=$(bc <<< "scale=2; ${freespace}/1000")
				# 				fsdsp="${fsgb} GB"
				# 			else
				# 				fsdsp="${freespace} MB"
				# 			fi
				# 			echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
				#
				# 		fi
				# 	fi
				# done

				echo
			done
		done
	fi
	########################
	if [ "${STATUS_ACTION}" != "check" ]; then
		echo 
		if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" == "gui" ]; then
			DIVIDER true
			ENTER2CONTINUE
		fi
	fi
	end=`date +%s`
	runtime=$((end-start))
	echo -e "Runtime: ${runtime}\n"
	[ "${STATUS_ACTION}" == "report" ] && echo "${log_start}~${runtime}" >> ${NM_LOGFOLDER}/status-check.scantimes
}

STATUS_NODE(){
	NODETYPE=${1}
	dockers=${NODETYPE}_DOCKERS_CHECK[@]
	hosts=${NODETYPE}_HOSTS[@]
	
	NTHOSTS=${NODETYPE}_HOSTS[@]
		
	if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${STATUS_ACTION}" == "services" ]); then
		GOFORCHECK=false;
	else
		GOFORCHECK=true;
	fi
	
	if [ ${GOFORCHECK} = true ]; then
	
		if [ "${STATUS_ACTION}" != "check" ]; then
			echo -e "${idsST[Bold]}"; DIVIDER . . ${DV_LEN}; DIVIDER . . ${DV_LEN}
			if [ "${STATUS_ACTION}" == "dockers" ]; then
				echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
			else
				echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
			fi
			DIVIDER . lightYellow ${DV_LEN}; echo -en "${idsST[Reset]}"
		fi
		for nip in "${!NTHOSTS}"; do
			if [ "${STATUS_ACTION}" != "check" ]; then
				echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}"
				[ "${STATUS_ACTION}" != "report" ] && echo -en " - Verifying Host..."
			fi
			
			if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
				NCMD=''; LH='- localhost'
			else
				NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH=''
			fi
			if [ "${NCMD}" != "" ]; then
				checkhost=$(CHECK_HOST ${nip})
				[ "${checkhost}" != "false" ] && checkhostssl=$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1)
			else
				checkhost=true
				checkhostssl=ok
			fi
			if [ "${checkhost}" != "false" ] && [ "${checkhostssl}" == "ok" ]; then
				[ "${NODETYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage=""
				

				if [ "${STATUS_ACTION}" != "check" ]; then
					if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then
						if [ "${NODETYPE}" == "OFW" ]; then
							uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1)
							if [ ${#uptime} -ge 6 ]; then
								uptime="up $(echo $uptime | cut -d: -f1) days, $(echo $uptime | cut -d: -f2) hours, $(echo $uptime | cut -d: -f3) minutes"
							elif [ ${#uptime} -ge 3 ]; then
								uptime="up $(echo $uptime | cut -d: -f1) hours, $(echo $uptime | cut -d: -f2) minutes"
							else
								uptime="up ${uptime} minutes"
							fi
							
						else
							uptime=$(${NCMD} uptime -p)
						fi
						[ "${STATUS_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
					else
						echo -e "${idsCL[Default]}"
					fi
					
					DIVIDER false lightCyan ${DV_LEN}
				fi
				
				if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
					if [ -f ${NM_TMPFOLDER}/${nip}.sent ]; then
						[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
						SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
					fi
					rm -f ${NM_TMPFOLDER}/${nip}.*
				fi
				
				########################
				## SERVICES CHECK
				########################						
				if [ "${STATUS_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then
					echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}"
					DIVIDER false green `expr ${DV_LEN} - 10`
					
					srvcs=${NODETYPE}_SERVICES_CHECK[@];
					srvcstotest="$(join_by " " ${!srvcs})"
					[ "${NODETYPE}" != "OFW" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}"`)
					sr=0
					for srvc in "${!srvcs}"; do
						if [ "${STATUS_ACTION}" != "report" ]; then
							c=0; cw=${PRI_CW}; spc=''
							spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
							until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
						else
							spc=' '
						fi
						tmp=${NM_SINGLESRVR_SERVICES[${NODETYPE}]}
						if [[ ${tmp[@]} =~ ${srvc} ]]; then
							if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then
								SSCHK=false
								rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
							else
								SSCHK=primary
							fi
						else
							if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then
								SSCHK=false
							elif [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]); then
								SSCHK=false
							else
								SSCHK=true
							fi
						fi
												
						if [ ${SSCHK} != false ]; then
							[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
							if [ "${NODETYPE}" != "OFW" ]; then
								srvctest=${srvctst[$sr]}
								if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then
									[ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false
								elif [ "${srvctest}" != "active" ] && [ "${srvc}" == "mariadb" ]; then
									[ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false
								elif [ "${srvctest}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then
									mysqlgo=true
								fi
							else
								if [ "$(ssh -o ConnectTimeout=3 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
									srvctest=active
								else
									srvctest=notactive
								fi
							fi
							if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
								if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
									[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running"
									touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
									# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
									echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
									[ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} &
									
								else
									[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})"
									if ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then
										[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
										touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
										SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is down" 1
										touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent
									fi
								fi
							else
								[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
								if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
									if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then
										[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
										SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' '${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is back UP! It was down for $(SHOW_TIME ${toterrtime})"
										echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
									fi

								fi
								if [[ "${NM_SINGLESRVR_SERVICES[${NODETYPE}]}" = *"${srvc}"* ]]; then
									for tnip in "${!NTHOSTS}"; do
										rm -f ${NM_TMPFOLDER}/${tnip}~${srvc}.* &
									done
								else
									rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* &
								fi
							fi
							[ "${mysqlgo}" == "true" ] && unset mysqlgo
							[ "${SSCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}"
							
						elif [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] || ([ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ])); then
							echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled${idsCL[Default]}"
							
						else
							echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}"
							
						fi
						sr=`expr $sr + 1`
					done
					
				fi
				
				########################
				## DOCKER CHECK
				########################
				if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" != "services" ]; then
					if [ "${STATUS_ACTION}" != "check" ]; then
						[ "${STATUS_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo
						echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}"
						DIVIDER false green `expr ${DV_LEN} - 10`
					fi
					for docker in "${!dockers}"; do
						if [ "${STATUS_ACTION}" != "report" ]; then
							c=0; cw=${PRI_CW}; spc=''
							spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
							until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
						else
							spc=' '
						fi
						tmp=${NM_SINGLESRVR_DOCKERS[${NODETYPE}]}
						if [[ ${tmp[@]} =~ ${docker} ]]; then
							if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then
								SDCHK=false
								rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
							else
								SDCHK=primary
							fi
						else
							SDCHK=true
						fi
						if [ ${SDCHK} != false ]; then
							[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
						
							if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
								if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ]; then
									
									if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
										touch ${NM_TMPFOLDER}/${nip}~${docker}.down
										echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, will try and fix in 1min" >> ${NM_LOGFILE}
										echo -en "${idsCL[Red]}Not Running - will try and fix in 1min!"
										
									elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) -gt 50 ]; then
										[ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
										if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
											${NCMD} docker start ${docker} >/dev/null 2>&1
											echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
										else
											${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
											echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
										fi
										touch ${NM_TMPFOLDER}/${nip}~${docker}.fix
										sleep 10s
										if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
											[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Green]}Running - Fixed"
											[ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed"
											rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
											echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}

										else
											if [ "${STATUS_ACTION}" != "check" ]; then
												[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
												SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1
												touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
												touch ${NM_TMPFOLDER}/${nip}~${docker}.down
												echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
												
											fi
										fi										
										
									elif [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then
										toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
										[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
										touch ${NM_TMPFOLDER}/${nip}~${docker}.down
										SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_SERVICE_DESC[${docker}]} - is down" 1
										touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
										echo -en "${idsCL[Red]}Not Running, could not fix earlier, FIX MX!"
									fi
									
								else
									echo -en "${idsCL[Yellow]}Disabled - Skipping"
								fi
							else
								if [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ]; then
									[ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed"
									[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Green]}Running - Fixed"
								else
									[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
								fi
								if [[ "${NM_SINGLESRVR_DOCKERS[${NODETYPE}]}" = *"${docker}"* ]]; then
									for tnip in "${!NTHOSTS}"; do
										rm -f ${NM_TMPFOLDER}/${tnip}~${docker}.* &
									done
								else
									rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* &
								fi
							fi
							[ "${SDCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}"
						else
							echo -e "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}"
						fi
					done
				fi
			
			else
				[ "${STATUS_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}"
				rm -f ${NM_TMPFOLDER}/${nip}~*
				if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
					touch ${NM_TMPFOLDER}/${nip}.down
					if [ "${STATUS_ACTION}" != "check" ]; then
						if [ "${checkhost}" != "false" ]; then
							echo -e "${idsCL[Red]} - Node is online, but SSH is down!${idsCL[Default]}"
						else
							echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
						fi
					fi
					# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
					[ "${checkhostssl}" != "ok" ] && echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
				else
					if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) -gt 60 ]; then
						mv ${NM_TMPFOLDER}/${nip}.down ${NM_TMPFOLDER}/${nip}.errtime
						toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
						errtime=${toterrtime}
						if [ "${checkhost}" != "false" ]; then
							SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is online, but SSH is down" 1
						else
							SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
						fi
						touch ${NM_TMPFOLDER}/${nip}.sent
					else
						errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
						[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=${errtime}
					fi

					if [ "${STATUS_ACTION}" != "check" ]; then
						if [ "${checkhost}" != "false" ]; then
							echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
						else
							echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
						fi
					fi
					if [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then
						if [ "${checkhost}" != "false" ]; then
							SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSL has been down for $(SHOW_TIME ${toterrtime})" 1
						else
							SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
						fi
						touch ${NM_TMPFOLDER}/${nip}.sent
					fi
					touch ${NM_TMPFOLDER}/${nip}.down
				fi

			fi
			[ "${STATUS_ACTION}" != "check" ] && echo
		done
		
	fi #GOFORCHECK
	
	touch ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.done
		
}

REPLCHECK(){
	rcheck=${1}
	nip=${2}
	PH_CMD=${3}
	
	# if [ "${4}" != "check" ]; then
	# 	timeout=`date --date='1 minutes' +%s`
	# 	# timeout=`date --date='30 seconds' +%s`
	# else
	# 	timeout=`date --date='2 minutes' +%s`
	# fi
	timeout=`date --date='1 minutes' +%s`
	# timeout=`date --date='30 seconds' +%s`
	
	checked=false
	until [ "${checked}" == "" ]; do
		if [ "${PH_CMD}" == "" ]; then
			ssh -q root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.${STATUS_START//-/}.repl ]] && checked=`ssh -o ConnectTimeout=3 root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/test.${STATUS_START//-/}.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/test.${STATUS_START//-/}.repl`
		else
			ssh -q root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.${STATUS_START//-/}.repl ]] && checked="`${PH_CMD} \"ssh -o ConnectTimeout=3 root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/test.${STATUS_START//-/}.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/test.${STATUS_START//-/}.repl\"`"
		fi
		[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
	done
	[ "${timeout}" == "true" ] && touch ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.good
}

REPLSTART(){
	NODETYPE=${1}
	PH=${NODETYPE}_HOSTS[0]
	if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
		PH_CMD="ssh root@${!PH}"
		var=${NODETYPE}_REPL_CHECK[@]
		if [ ! -z ${!var+x} ]; then
			for rcheck in "${!var}"; do
				${PH_CMD} rm -f ${NM_FOLDER}/test.${STATUS_START//-/}.repl
				daterun=`date +%Y-%m-%d-%H-%M-%S`
				${PH_CMD} "echo -e \"Replcation-Test\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/test.${STATUS_START//-/}.repl" &
				${PH_CMD} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' &
			done
		fi
	fi
}

STATUS_SCANTIMES(){
	scandefault=200
	while [ $# -gt 0 ]; do
		case "${1}" in
			-l|-last) LAST=${2};;
			-h|-help|--help)
				echo -e "Usage: ${idsCL[Yellow]}[nodemgmt or nmg] scantimes {opt}${idsCL[Default]} {"
				width=15
				printf "%-${width}s- %s\n" "    -l|-last" "(number of last scan times to average from, defaults to ${scandefault})"
				echo -e "}\n"
				exit 0;;
		esac
		shift
	done
	[ "${LAST}" == "" ] && LAST=${scandefault}
	
	echo -en "${idsCL[Yellow]}Pulling data and calculating average scan times for the last '${LAST}' scans ... "

	lastscantime=$(tail -n 1 ${NM_LOGFOLDER}/status-check.scantimes)
	lastscantime=${lastscantime#*~}
	scantimes=$(tail -n ${LAST} ${NM_LOGFOLDER}/status-check.scantimes)
	IFS=$'\n'
	read -rd '' -a scantimes <<<"${scantimes}"
	unset IFS
	scantimes_total_norm=0; scansfound_norm=0
	scantimes_total_repl=0; scansfound_repl=0
	scantimes_total_xtr=0;  scansfound_xtr=0
	for scantime in "${scantimes[@]}"; do
		[[ "${scantime}" = *"~"* ]] && scantime=${scantime#*~}
		if [ ${scantime} -le 35 ]; then
			scantimes_total_norm=$(bc <<< "scale=2; ${scantimes_total_norm}+${scantime}")
			((scansfound_norm++))
		elif [ ${scantime} -le 60 ]; then
			scantimes_total_repl=$(bc <<< "scale=2; ${scantimes_total_repl}+${scantime}")
			((scansfound_repl++))
		else
			scantimes_total_xtr=$(bc <<< "scale=2; ${scantimes_total_xtr}+${scantime}")
			((scansfound_xtr++))
		fi
	done
	
	scantimes_total=`expr ${scantimes_total_norm} + ${scantimes_total_repl} + ${scantimes_total_xtr}`
	scansfound=`expr ${scansfound_norm} + ${scansfound_repl} + ${scansfound_xtr}`
		
	[ ${scansfound_norm} -eq 0 ] && average_norm=0 || average_norm=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_total_norm}/${scansfound_norm}") 0)
	[ ${scansfound_repl} -eq 0 ] && average_repl=0 || average_repl=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_total_repl}/${scansfound_repl}") 0)
	[ ${scansfound_xtr} -eq 0 ] && average_xtr=0 || average_xtr=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_total_xtr}/${scansfound_xtr}") 0)
	[ ${scansfound} -eq 0 ] && average=0 || average=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_total}/${scansfound}") 0)
	
	
	cw=`expr $(echo "${#average_norm} ${#average_repl} ${#average_xtr} ${#average}" | grep -oE '[0-9]+' | sort -n | tail -n 1) + 2`
	average_norm_disp=$(SHOW_TIME ${average_norm} s)
	c=0; spca=''; spc1=`expr ${cw} - ${#average_norm}`; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
	average_repl_disp=$(SHOW_TIME ${average_repl} s)
	c=0; spcr=''; spc1=`expr ${cw} - ${#average_repl}`; until [ $c = ${spc1} ]; do spcr="${spcr} "; c=`expr $c + 1`; done
	average_xtr_disp=$(SHOW_TIME ${average_xtr} s)
	c=0; spcx=''; spc1=`expr ${cw} - ${#average_xtr}`; until [ $c = ${spc1} ]; do spcx="${spcx} "; c=`expr $c + 1`; done
	average_disp=$(SHOW_TIME ${average} s)
	c=0; spcT=''; spc1=`expr ${cw} - ${#average}`; until [ $c = ${spc1} ]; do spct="${spct} "; c=`expr $c + 1`; done
	
	lastscantime_disp=$(SHOW_TIME ${lastscantime} s)
	c=0; spcb=''; spc1=`expr ${cw} - ${#lastscantime}`; until [ $c = ${spc1} ]; do spcb="${spcb} "; c=`expr $c + 1`; done
	
	
	[ -f ${NM_TMPFOLDER}/.replcheck ] && last_replcheck=$(date -d @$(stat -c %Y ${NM_TMPFOLDER}/.replcheck) "+%Y-%m-%d %H:%M:%S") || last_replcheck=""
	[ -f ${NM_LOGFOLDER}/status-check.scantimes ] && last_scancheck=$(date -d @$(stat -c %Y ${NM_LOGFOLDER}/status-check.scantimes) "+%Y-%m-%d %H:%M:%S") || last_scancheck=""
	
	echo -en "\r\033[K"
	echo -e "${idsCL[LightCyan]}Scan Type		# of Scans   Average Scantime${idsCL[Default]}"
	DIVIDER . lightCyan 60
	echo -e "${idsCL[Green]}Average for last	${scansfound}	: ${idsCL[LightGreen]}(${average}s)${spct}${idsCL[Green]}${average_disp}${idsCL[Default]}"
	DIVIDER . lightCyan 60
	echo -e "${idsCL[Cyan]}  - Normal runs  	${scansfound_norm}	: ${idsCL[LightCyan]}(${average_norm}s)${spca}${idsCL[Cyan]}${average_norm_disp}${idsCL[Default]}"
	echo -e "${idsCL[Cyan]}  - w/Replication	${scansfound_repl}	: ${idsCL[LightCyan]}(${average_repl}s)${spcr}${idsCL[Cyan]}${average_repl_disp}${idsCL[Default]}"
	echo -e "${idsCL[Cyan]}  - Extreme runs	${scansfound_xtr}	: ${idsCL[LightCyan]}(${average_xtr}s)${spcx}${idsCL[Cyan]}${average_xtr_disp}${idsCL[Default]}"
	echo
	echo -e "${idsCL[LightCyan]}Last Scan Time         	: ${idsCL[LightGreen]}(${lastscantime}s)${spcb}${idsCL[Green]}${lastscantime_disp}${idsCL[Default]}\n"
	echo -e "${idsCL[LightCyan]}Last Scan Check	    	: ${idsCL[LightGreen]}${last_scancheck}${idsCL[Default]}\n"
	echo -e "${idsCL[LightCyan]}Last Replication Check	: ${idsCL[LightGreen]}${last_replcheck}${idsCL[Default]}\n"
	
	if [ ${scansfound} -le 500 ]; then
		echo -en "\n${idsCL[LightCyan]}List out these '${scansfound}' scan times? (y/N): ${idsCL[Default]}"
		read -n1 choice
		if [ "${choice^^}" == "Y" ]; then
			echo -en "\r\033[K"
			lastscansnum=${scansfound}
			last_scantimes=$(tail -n ${lastscansnum} ${NM_LOGFOLDER}/status-check.scantimes)
			IFS=$'\n'
			read -rd '' -a last_scantimes <<<"${last_scantimes}"
			unset IFS
			echo -e "${idsCL[LightCyan]}Last ${lastscansnum} Scan Times\n${idsCL[Green]}------------------------------------------${idsCL[Default]}"
			for scantime in "${last_scantimes[@]}"; do
				if [[ "${scantime}" = *"~"* ]]; then
					dt="${scantime%~*}"
					scantime=${scantime#*~}
				else
					dt="		"
				fi
				echo -e "${idsCL[White]} ${dt}	(${scantime}s)	${idsCL[Yellow]}$(SHOW_TIME ${scantime} s)${idsCL[Default]}"
			done
		fi
	fi

	echo -e "\n"
}