From 872b3a75584b1e3a995bad32da2fc8db9cbc64bb Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Tue, 26 Mar 2019 23:26:31 -0500
Subject: [PATCH] Update nodemgmt-scripts.sh

---
 nodemgmt-scripts.sh | 183 +++++++++++++++-----------------------------
 1 file changed, 63 insertions(+), 120 deletions(-)

diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index 9da301a3..b8d686ad 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -42,7 +42,6 @@ STATUS(){
 		for NTYPE in "${NODE_TYPES[@]}"; do
 			PH=${NTYPE}_HOSTS[0]
 			PH_CMD="ssh root@${!PH}"
-			echo ": ${PH_CMD}"
 			var=${NTYPE}_REPL_CHECK[@]
 			if [ ! -z ${!var+x} ]; then
 				for rcheck in "${!var}"; do
@@ -229,137 +228,81 @@ STATUS(){
 			echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
 			DIVIDER; echo -e "${idsST[Reset]}"
 		fi
-		if [ ! -z ${MYSQL_REPL_CHECK+x} ]; then
-			nid=1
-			for nip in "${MYSQL_HOSTS[@]}"; do
-				if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${MYSQL_HOSTS[0]}" ]; then
-					if [ "${ST_ACTION}" != "check" ]; then
-						echo -e " ${idsCL[LightCyan]}MySQL-Node1 (${MYSQL_HOSTS[0]}) <--> ${idsST[Bold]}MySQL-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
-						DIVIDER false green
-					fi
-					checkhost=$(CHECK_HOST ${nip})
-					if [ "${checkhost}" != "false" ]; then
-						if [ -f ${FOLDER}/${nip}.down ]; then
-							rm -f ${FOLDER}/${nip}.down
-							if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} is back UP!" | mail -s "MySQL-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
+		
+		
+		for NTYPE in "${NODE_TYPES[@]}"; do
+			PH=${NTYPE}_HOSTS[0]
+			PH_CMD="ssh root@${!PH}"
+			var=${NTYPE}_REPL_CHECK[@]
+			if [ ! -z ${!var+x} ]; then
+				nid=1
+				
+				var=${NTYPE}_HOSTS[@]
+				for nip in "${!var}"; do
+					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!var}" ]; then
+						if [ "${ST_ACTION}" != "check" ]; then
+							echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
+							DIVIDER false green
 						fi
-						for rcheck in "${MYSQL_REPL_CHECK[@]}"; do
-							if [ "${ST_ACTION}" != "check" ]; then
-								c=0; cw=22; spc=''
-								spc1=${cw}-${#REPL_DESC[${rcheck}]}
-								timeout=`date --date='20 seconds' +%s`
-								until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
-								echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
+						checkhost=$(CHECK_HOST ${nip})
+						if [ "${checkhost}" != "false" ]; then
+							if [ -f ${FOLDER}/${nip}.down ]; then
+								rm -f ${FOLDER}/${nip}.down
+								if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP!" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
 							fi
-							checked=false
-							until [ "${checked}" = "" ]; do
-								if [ "${NHCMD}" = "" ]; then
-									checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
+							for rcheck in "${!var}"; do
+								if [ "${ST_ACTION}" != "check" ]; then
+									c=0; cw=22; spc=''
+									spc1=${cw}-${#REPL_DESC[${rcheck}]}
+									timeout=`date --date='20 seconds' +%s`
+									until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+									echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
+								fi
+								checked=false
+								until [ "${checked}" = "" ]; do
+									if [ "${PH_CMD}" = "" ]; then
+										checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
+									else
+										checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
+									fi
+									if [ "`date +%s`" -gt "$timeout" ]; then
+										timeout=true
+										break
+									fi
+								done
+								if [ "${timeout}" = "true" ]; then
+									if [ "${ST_ACTION}" != "check" ]; then
+										echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
+									fi
+									if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+										touch ${FOLDER}/${nip}-${rcheck}.down
+										if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi
+									fi
 								else
-									checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
+									if [ "${ST_ACTION}" != "check" ]; then
+										echo -e "${idsCL[Green]}Good${idsCL[Default]}"
+									fi
+									if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+										rm -f ${FOLDER}/${nip}-${rcheck}.down
+										if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi
+									fi
 								fi
-								if [ "`date +%s`" -gt "$timeout" ]; then
-									timeout=true
-									break
-								fi
-							done
-							if [ "${timeout}" = "true" ]; then
-								if [ "${ST_ACTION}" != "check" ]; then
-									echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
-								fi
-								if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-									touch ${FOLDER}/${nip}-${rcheck}.down
-									if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'MySQL-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi
-								fi
-							else
-								if [ "${ST_ACTION}" != "check" ]; then
-									echo -e "${idsCL[Green]}Good${idsCL[Default]}"
-								fi
-								if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-									rm -f ${FOLDER}/${nip}-${rcheck}.down
-									if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'MySQL-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi
-								fi
-							fi
 						
-						done
-					else
-						if [ "${ST_ACTION}" != "check" ]; then
-							echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
-						fi
-						if [ ! -f ${FOLDER}/${nip}.down ]; then
-							touch ${FOLDER}/${nip}.down
-							if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} is down" | mail -s "MySQL-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}; fi
-						fi
-					fi
-					if [ "${ST_ACTION}" != "check" ]; then echo; fi
-				fi
-				nid=`expr $nid + 1`
-			done
-		fi
-		nid=1
-		for nip in "${WEB_HOSTS[@]}"; do
-			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${WEB_HOSTS[0]}" ]; then
-				if [ "${ST_ACTION}" != "check" ]; then
-					echo -e " ${idsCL[LightCyan]}Webserver-Node1 (${WEB_HOSTS[0]}) <--> ${idsST[Bold]}Webserver-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
-					DIVIDER false green
-				fi
-				checkhost=$(CHECK_HOST ${nip})
-				if [ "${checkhost}" != "false" ]; then
-					if [ -f ${FOLDER}/${nip}.down ]; then
-						rm -f ${FOLDER}/${nip}.down
-						if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Webserver-Node${nid} is back UP!" | mail -s "Webserver-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
-					fi
-					for rcheck in "${WEB_REPL_CHECK[@]}"; do
-						if [ "${ST_ACTION}" != "check" ]; then
-							c=0; cw=22; spc=''
-							spc1=${cw}-${#REPL_DESC[${rcheck}]}
-							timeout=`date --date='20 seconds' +%s`
-							until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
-							echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
-						fi
-						checked=false
-						until [ "${checked}" = "" ]; do
-							if [ "${WHCMD}" = "" ]; then
-								checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
-							else
-								checked="`${WHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
-							fi
-							if [ "`date +%s`" -gt "$timeout" ]; then
-								timeout=true
-								break
-							fi
-						done
-						if [ "${timeout}" = "true" ]; then
-							if [ "${ST_ACTION}" != "check" ]; then
-								echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
-							fi
-							if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-								touch ${FOLDER}/${nip}-${rcheck}.down
-								if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'Webserver-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi
-							fi
+							done
 						else
 							if [ "${ST_ACTION}" != "check" ]; then
-								echo -e "${idsCL[Green]}Good${idsCL[Default]}"
+								echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
 							fi
-							if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-								rm -f ${FOLDER}/${nip}-${rcheck}.down
-								if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'Webserver-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi
+							if [ ! -f ${FOLDER}/${nip}.down ]; then
+								touch ${FOLDER}/${nip}.down
+								if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}; fi
 							fi
 						fi
-						
-					done
-				else
-					if [ "${ST_ACTION}" != "check" ]; then
-						echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
+						if [ "${ST_ACTION}" != "check" ]; then echo; fi
 					fi
-					if [ ! -f ${FOLDER}/${nip}.down ]; then
-						touch ${FOLDER}/${nip}.down
-						if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Webserver-Node${nid} is down" | mail -s "Webserver-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}; fi
-					fi
-				fi
-				if [ "${ST_ACTION}" != "check" ]; then echo; fi
+					nid=`expr $nid + 1`
+				done
 			fi
-			nid=`expr $nid + 1`
 		done
 	fi