From 136b4dee5242f18872a998dffff1d64b782ac9de Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Thu, 7 Feb 2019 14:39:46 -0600
Subject: [PATCH] Update nodemgmt-scripts.sh

---
 nodemgmt-scripts.sh | 99 +++++++++++++++++++++++++++------------------
 1 file changed, 60 insertions(+), 39 deletions(-)

diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index 72436666..ceb8271f 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -80,6 +80,7 @@ STATUS() {
 		DIVIDER false green
 		checkhost=$(CHECK_HOST ${nip})
 		if [ "${checkhost}" != "false" ]; then
+			rm -f ${FOLDER}/${nip}.down
 			for srvc in "${NODE_SERVICES[@]}"
 			do
 				c=0; cw=22; spc=''
@@ -96,6 +97,7 @@ STATUS() {
 			done
 		else
 			echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
+			touch ${FOLDER}/${nip}.down
 		fi
 		echo
 		nid=`expr $nid + 1`
@@ -112,6 +114,7 @@ STATUS() {
 			DIVIDER false green
 			checkhost=$(CHECK_HOST ${nip})
 			if [ "${checkhost}" != "false" ]; then
+				rm -f ${FOLDER}/${nip}.down
 				for rcheck in "${REPL_CHECK[@]}"; do
 					c=0; cw=22; spc=''
 					spc1=${cw}-${#REPL_DESC[${rcheck}]}
@@ -142,6 +145,7 @@ STATUS() {
 				done
 			else
 				echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
+				touch ${FOLDER}/${nip}.down
 			fi
 			echo
 		fi
@@ -186,55 +190,72 @@ STATUS-CHECK(){
 		fi
 		if [ -z ${LOCAL_SERVICES+x} ] || [ "${2}" = "all" ]; then
 			nid=1
-			for nip in "${NODE_HOSTS[@]}"
-			do
-				if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
-					else NCMD="ssh root@${nip}"
-				fi
-				for srvc in "${NODE_SERVICES[@]}"
-				do
-					if [ $(${NCMD} pgrep ${srvc} | wc -l) -lt "1" ]; then
-						echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
-						touch ${FOLDER}/${nip}-${srvc}.down
-					elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
-						echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
-						rm -f ${FOLDER}/${nip}-${srvc}.down
+			for nip in "${NODE_HOSTS[@]}"; do
+				checkhost=$(CHECK_HOST ${nip})
+				if [ "${checkhost}" != "false" ]; then
+					if [ -f ${FOLDER}/${nip}.down ]; then
+						rm -f ${FOLDER}/${nip}.down
+						echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
 					fi
-				done
+					if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
+						else NCMD="ssh root@${nip}"
+					fi
+					for srvc in "${NODE_SERVICES[@]}"
+					do
+						if [ $(${NCMD} pgrep ${srvc} | wc -l) -lt "1" ]; then
+							echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+							touch ${FOLDER}/${nip}-${srvc}.down
+						elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
+							echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+							rm -f ${FOLDER}/${nip}-${srvc}.down
+						fi
+					done
+				elif [ ! -f ${FOLDER}/${nip}.down ]; then
+					touch ${FOLDER}/${nip}.down
+					echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
+				fi
 				nid=`expr $nid + 1`
 			done
 		fi
 		##REPLICATION CHECK
 		nid=1
-		for nip in "${NODE_HOSTS[@]}"
-		do
+		for nip in "${NODE_HOSTS[@]}"; do
 			if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
 			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
-				for rcheck in "${REPL_CHECK[@]}"; do
-					timeout=`date --date='20 seconds' +%s`
-					checked=false
-					until [ "${checked}" = "" ]; do
-						if [ "${NCMD}" = "" ]; then
-							checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
-						else
-							checked="`ssh root@10.5.10.51 \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
-						fi
-						if [ "`date +%s`" -gt "$timeout" ]; then
-							timeout=true
-							break
+				checkhost=$(CHECK_HOST ${nip})
+				if [ "${checkhost}" != "false" ]; then
+					if [ -f ${FOLDER}/${nip}.down ]; then
+						rm -f ${FOLDER}/${nip}.down
+						echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
+					fi
+					for rcheck in "${REPL_CHECK[@]}"; do
+						timeout=`date --date='20 seconds' +%s`
+						checked=false
+						until [ "${checked}" = "" ]; do
+							if [ "${NCMD}" = "" ]; then
+								checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
+							else
+								checked="`ssh root@10.5.10.51 \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
+							fi
+							if [ "`date +%s`" -gt "$timeout" ]; then
+								timeout=true
+								break
+							fi
+						done
+						if [ "${timeout}" != "true" ]; then
+							if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+								rm -f ${FOLDER}/${nip}-${rcheck}.down
+								echo "Replicated folder is backl up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
+							fi
+						elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+							touch ${FOLDER}/${nip}-${rcheck}.down
+							echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
 						fi
 					done
-					if [ "${timeout}" != "true" ]; then
-						if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-							rm -f ${FOLDER}/${nip}-${rcheck}.down
-							echo "Replicated folder is backl up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
-						fi
-					elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-						touch ${FOLDER}/${nip}-${rcheck}.down
-						echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
-					fi
-				done
-				echo
+				elif [ ! -f ${FOLDER}/${nip}.down ]; then
+					touch ${FOLDER}/${nip}.down
+					echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
+				fi
 			fi
 			nid=`expr $nid + 1`
 		done