From a7093e6155f64ff3e27746bc0482df4c46e96251 Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Thu, 27 Apr 2023 08:57:12 -0500
Subject: [PATCH] Update nodemgmt-scripts.sh

---
 nodemgmt-scripts.sh | 309 ++++++++++++++++++++------------------------
 1 file changed, 138 insertions(+), 171 deletions(-)

diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index 78d67808..1a878ce5 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -25,6 +25,32 @@ STATUS(){
 		ST_ACTION=repl
 	fi
 	declare -i cw; declare -i spc1; declare -i c
+	
+	########################
+	if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
+		if [ "${ST_ACTION}" != "check" ]; then
+			echo -en "${idsCL[LightCyan]}Setting up replication checks for next status check ... ${idsCL[Default]}"
+		fi
+		for NTYPE in "${NODE_TYPES[@]}"; do
+			PH=${NTYPE}_HOSTS[0]
+			if [ ! -f ${FOLDER}/${!PH}.down ]; then
+				PH_CMD="ssh root@${!PH}"
+				var=${NTYPE}_REPL_CHECK[@]
+				if [ ! -z ${!var+x} ]; then
+					for rcheck in "${!var}"; do
+						# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
+						${PH_CMD} rm -f ${FOLDER}/test.repl
+						daterun=`date +%Y-%m-%d-%H-%M-%S`
+						${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" &
+					done
+				fi
+			fi
+		done
+		if [ "${ST_ACTION}" != "check" ]; then
+			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
+			echo
+		fi
+	fi
 
 	########################
 	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then
@@ -340,10 +366,113 @@ STATUS(){
 	########################
 	
 	if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
-		if [ ! -f ${FOLDER}/test.repl ]; then
-			touch ${FOLDER}/test.repl
+		if [ "${ST_ACTION}" != "check" ]; then
+			echo -e "${idsST[Bold]}"; DIVIDER
+			echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
+			DIVIDER; echo -e "${idsST[Reset]}"
+		fi
+	
+		for NTYPE in "${NODE_TYPES[@]}"; do
+			PH=${NTYPE}_HOSTS[0]
+			PH_CMD="ssh root@${!PH}"
+			var=${NTYPE}_REPL_CHECK[@]
+			if [ ! -z ${!var+x} ]; then
+				nid=1
+			
+				var=${NTYPE}_HOSTS[@]
+				for nip in "${!var}"; do
+				
+					# echo "CHECK: ${NTYPE} - ${nip}"
+				
+					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
+						if [ "${ST_ACTION}" != "check" ]; then
+							echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
+							DIVIDER false green
+						fi
+						
+						# checkhost=$(CHECK_HOST ${nip})
+						#if [ "${checkhost}" != "false" ]; then
+						if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
+							var2=${NTYPE}_REPL_CHECK[@]
+							# timeout=`date --date='20 seconds' +%s`
+							for rcheck in "${!var2}"; do
+								if [ "${ST_ACTION}" != "check" ]; then
+									if [ "${ST_ACTION}" != "report" ]; then
+										c=0; cw=30; spc=''
+										spc1=${cw}-${#REPL_DESC[${rcheck}]}
+										until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+									else
+										spc=' '
+									fi
+									echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
+									
+									timeout=`date --date='2 minutes' +%s`
+									
+									# if [ "${rcheck}" == "www" ]; then
+									#  	timeout=`date --date='2 minutes' +%s`
+									#  else
+									# 	timeout=`date --date='20 seconds' +%s`
+									#  fi
+								
+								else
+									timeout=`date --date='2 minutes' +%s`
+								fi
+								warn_timeout=`date --date='30 secs' +%s`
+								
+								checked=false
+								until [ "${checked}" = "" ]; do
+									if [ "${PH_CMD}" = "" ]; then
+										ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
+									else
+										ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
+									fi
+									if [ "`date +%s`" -gt "${timeout}" ]; then
+										timeout=true
+										break
+									elif [ "`date +%s`" -gt "${warn_timeout}" ]; then
+										echo -en " waiting for sync...  "
+										warn_timeout=`date --date='100 minutes' +%s`
+									fi
+								done
+								if [ "${timeout}" = "true" ]; then
+									if [ "${ST_ACTION}" != "check" ]; then
+										echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
+									fi
+									if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+										touch ${FOLDER}/${nip}-${rcheck}.down
+										SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
+										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
+									fi
+								else
+									if [ "${ST_ACTION}" != "check" ]; then
+										echo -e "${idsCL[Green]}Good${idsCL[Default]}"
+									fi
+									if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+										rm -f ${FOLDER}/${nip}-${rcheck}.down
+										SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
+										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
+									fi
+								fi
+				
+							done
+						else
+							if [ "${ST_ACTION}" != "check" ]; then
+								[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}"
+								[ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}"
+							fi
+						fi
+						if [ "${ST_ACTION}" != "check" ]; then echo; fi
+					fi
+					nid=`expr $nid + 1`
+				done
+			fi
+		done
+		##########################
+		# REMOVE REPL CHECK FILES
+		##########################
+		if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
 			if [ "${ST_ACTION}" != "check" ]; then
-				echo -en "${idsCL[LightCyan]}Setting up replication checks for next status check ... ${idsCL[Default]}"
+				echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
 			fi
 			#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
 			#LBHCMD="ssh root@${LB_HOSTS[0]}"
@@ -355,187 +484,25 @@ STATUS(){
 					var=${NTYPE}_REPL_CHECK[@]
 					if [ ! -z ${!var+x} ]; then
 						for rcheck in "${!var}"; do
-							# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
 							${PH_CMD} rm -f ${FOLDER}/test.repl
 							daterun=`date +%Y-%m-%d-%H-%M-%S`
 							if [ "${PH_CMD}" = "" ]; then
-								echo -e "${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
-								yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
+								rm -f ${FOLDER}/test.repl &
+								rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
 							else
-								${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl"
-								# ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
-								# yes | ${PH_CMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
+								${PH_CMD} rm -f ${FOLDER}/test.repl &
+								${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
 							fi
 						done
 					fi
 				fi
 			done
+			rm -Rf ${FOLDER}/test.repl
 			if [ "${ST_ACTION}" != "check" ]; then
 				echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
 				echo
 			fi
-			[ "${ST_ACTION}" != "repl" ] && CHECKFOLDERSYNC=false || CHECKFOLDERSYNC=true
-		else
-			rm -Rf ${FOLDER}/test.repl
-			if [ "${ST_ACTION}" != "check" ]; then
-				echo
-				echo -e "${idsCL[LightCyan]}Will verify folder replication this status check ... ${idsCL[Default]}"
-				CHECKFOLDERSYNC=true
-			fi
-		fi
-		
-		########################
-		
-		if [ "${CHECKFOLDERSYNC}" == true ]; then
-			if [ "${ST_ACTION}" != "check" ]; then
-				echo -e "${idsST[Bold]}"; DIVIDER
-				echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
-				DIVIDER; echo -e "${idsST[Reset]}"
-			fi
-		
-		
-			for NTYPE in "${NODE_TYPES[@]}"; do
-				#GOREPCHK=false;
-				#CHR=`date +"%H"`;
-				#if [ "${CHR}" -ge "12" ] && [ "${CHR}" -le "13" ] && [ "${NTYPE}" = "NC" ]; then
-				#	GOREPCHK=true;
-				#elif [ "${NTYPE}" != "NC" ]; then
-				#	GOREPCHK=true;
-				#fi
-				GOREPCHK=true;
-				if [ "${GOREPCHK}" = "true" ]; then
-					PH=${NTYPE}_HOSTS[0]
-					PH_CMD="ssh root@${!PH}"
-					var=${NTYPE}_REPL_CHECK[@]
-					if [ ! -z ${!var+x} ]; then
-						nid=1
-					
-						var=${NTYPE}_HOSTS[@]
-						for nip in "${!var}"; do
-						
-							# echo "CHECK: ${NTYPE} - ${nip}"
-						
-							if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
-								if [ "${ST_ACTION}" != "check" ]; then
-									echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
-									DIVIDER false green
-								fi
-								
-								# checkhost=$(CHECK_HOST ${nip})
-								#if [ "${checkhost}" != "false" ]; then
-								if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
-									var2=${NTYPE}_REPL_CHECK[@]
-									# timeout=`date --date='20 seconds' +%s`
-									for rcheck in "${!var2}"; do
-										if [ "${ST_ACTION}" != "check" ]; then
-											if [ "${ST_ACTION}" != "report" ]; then
-												c=0; cw=30; spc=''
-												spc1=${cw}-${#REPL_DESC[${rcheck}]}
-												until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
-											else
-												spc=' '
-											fi
-											echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
-											
-											timeout=`date --date='2 minutes' +%s`
-											
-											# if [ "${rcheck}" == "www" ]; then
-											#  	timeout=`date --date='2 minutes' +%s`
-											#  else
-											# 	timeout=`date --date='20 seconds' +%s`
-											#  fi
-										
-										else
-											timeout=`date --date='2 minutes' +%s`
-										fi
-										warn_timeout=`date --date='30 secs' +%s`
-										
-										checked=false
-										until [ "${checked}" = "" ]; do
-											if [ "${PH_CMD}" = "" ]; then
-												ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
-											else
-												ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
-											fi
-											if [ "`date +%s`" -gt "${timeout}" ]; then
-												timeout=true
-												break
-											elif [ "`date +%s`" -gt "${warn_timeout}" ]; then
-												echo -en " waiting for sync...  "
-												warn_timeout=`date --date='100 minutes' +%s`
-											fi
-										done
-										if [ "${timeout}" = "true" ]; then
-											if [ "${ST_ACTION}" != "check" ]; then
-												echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
-											fi
-											if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-												touch ${FOLDER}/${nip}-${rcheck}.down
-												SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
-												echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
-											fi
-										else
-											if [ "${ST_ACTION}" != "check" ]; then
-												echo -e "${idsCL[Green]}Good${idsCL[Default]}"
-											fi
-											if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-												rm -f ${FOLDER}/${nip}-${rcheck}.down
-												SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
-												echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
-											fi
-										fi
-						
-									done
-								else
-									if [ "${ST_ACTION}" != "check" ]; then
-										[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}"
-										[ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}"
-									fi
-								fi
-								if [ "${ST_ACTION}" != "check" ]; then echo; fi
-							fi
-							nid=`expr $nid + 1`
-						done
-					fi
-				fi
-			done
-			##########################
-			# REMOVE REPL CHECK FILES
-			##########################
-			if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
-				if [ "${ST_ACTION}" != "check" ]; then
-					echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
-				fi
-				#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
-				#LBHCMD="ssh root@${LB_HOSTS[0]}"
-				#WHCMD="ssh root@${WEB_HOSTS[0]}"
-				for NTYPE in "${NODE_TYPES[@]}"; do
-					PH=${NTYPE}_HOSTS[0]
-					if [ ! -f ${FOLDER}/${!PH}.down ]; then
-						PH_CMD="ssh root@${!PH}"
-						var=${NTYPE}_REPL_CHECK[@]
-						if [ ! -z ${!var+x} ]; then
-							for rcheck in "${!var}"; do
-								${PH_CMD} rm -f ${FOLDER}/test.repl
-								daterun=`date +%Y-%m-%d-%H-%M-%S`
-								if [ "${PH_CMD}" = "" ]; then
-									rm -f ${FOLDER}/test.repl
-									rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
-								else
-									${PH_CMD} rm -f ${FOLDER}/test.repl
-									${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
-								fi
-							done
-						fi
-					fi
-				done
-				rm -Rf ${FOLDER}/test.repl
-				if [ "${ST_ACTION}" != "check" ]; then
-					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
-					echo
-				fi
-			fi	
-		fi
+		fi	
 	fi
 	
 	########################