From e06736f0a77cb6f481d4c4ad0148f7893b0cdbfa Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Tue, 26 Feb 2019 23:16:20 -0600
Subject: [PATCH] Update nodemgmt-scripts.sh

---
 nodemgmt-scripts.sh | 78 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 72 insertions(+), 6 deletions(-)

diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index 36112d46..349c7f7b 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -37,7 +37,7 @@ STATUS(){
 			echo
 		fi
 		if [ ! -z ${LOCAL_SERVICES+x} ];
-			then NHCMD="ssh root@${PRIMARYHOST}"
+			then NHCMD="ssh root@${NODE_HOSTS[0]}"
 			else NHCMD=''
 		fi
 		for rcheck in "${REPL_CHECK[@]}"; do
@@ -307,7 +307,7 @@ STATUS(){
 		fi
 		nid=1
 		for nip in "${NODE_HOSTS[@]}"; do
-			if [ "${nip}" = "${PRIMARYHOST}" ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
+			if [ "${nip}" = "${NODE_HOSTS[0]}" ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
 			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
 				if [ "${ST_ACTION}" != "check" ]; then
 					echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-MySQL${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
@@ -319,7 +319,7 @@ STATUS(){
 						rm -f ${FOLDER}/${nip}.down
 						echo "iDS-MySQL${nid} is back UP!" | mail -s "iDS-MySQL${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
 					fi
-					for rcheck in "${REPL_CHECK[@]}"; do
+					for rcheck in "${NODE_REPL_CHECK[@]}"; do
 						if [ "${ST_ACTION}" != "check" ]; then
 							c=0; cw=22; spc=''
 							spc1=${cw}-${#REPL_DESC[${rcheck}]}
@@ -371,6 +371,72 @@ STATUS(){
 			fi
 			nid=`expr $nid + 1`
 		done
+		nid=1
+		for nip in "${WEB_HOSTS[@]}"; do
+			if [ "${nip}" = "${WEB_HOSTS[0]}" ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
+			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
+				if [ "${ST_ACTION}" != "check" ]; then
+					echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Webserver${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
+					DIVIDER false green
+				fi
+				checkhost=$(CHECK_HOST ${nip})
+				if [ "${checkhost}" != "false" ]; then
+					if [ -f ${FOLDER}/${nip}.down ]; then
+						rm -f ${FOLDER}/${nip}.down
+						echo "iDS-Webserver${nid} is back UP!" | mail -s "iDS-Webserver${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
+					fi
+					for rcheck in "${WEB_REPL_CHECK[@]}"; do
+						if [ "${ST_ACTION}" != "check" ]; then
+							c=0; cw=22; spc=''
+							spc1=${cw}-${#REPL_DESC[${rcheck}]}
+							timeout=`date --date='20 seconds' +%s`
+							until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+							echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
+						fi
+						checked=false
+						until [ "${checked}" = "" ]; do
+							if [ "${NHCMD}" = "" ]; then
+								checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
+							else
+								checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
+							fi
+							if [ "`date +%s`" -gt "$timeout" ]; then
+								timeout=true
+								break
+							fi
+						done
+						if [ "${timeout}" = "true" ]; then
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
+							fi
+							if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+								touch ${FOLDER}/${nip}-${rcheck}.down
+								echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Webserver${nid}'" ${STATUS_CHECK_EMAIL}
+							fi
+						else
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Green]}Good${idsCL[Default]}"
+							fi
+							if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+								rm -f ${FOLDER}/${nip}-${rcheck}.down
+								echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Webserver${nid}'" ${STATUS_CHECK_EMAIL}
+							fi
+						fi
+						
+					done
+				else
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
+					fi
+					if [ ! -f ${FOLDER}/${nip}.down ]; then
+						touch ${FOLDER}/${nip}.down
+						echo "iDS-Webserver${nid} is down" | mail -s "iDS-Webserver${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
+					fi
+				fi
+				if [ "${ST_ACTION}" != "check" ]; then echo; fi
+			fi
+			nid=`expr $nid + 1`
+		done
 	fi
 	
 	if [ "${ST_ACTION}" != "check" ]; then
@@ -388,7 +454,7 @@ STATUS(){
 # STATUS-CHECK(){
 # 	start=`date +%s`
 # 	if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
-# 		if [ ! -z ${LOCAL_SERVICES+x} ]; then NHCMD="ssh root@${PRIMARYHOST}"
+# 		if [ ! -z ${LOCAL_SERVICES+x} ]; then NHCMD="ssh root@${NODE_HOSTS[0]}"
 # 		else NHCMD=''; fi
 # 		# for rcheck in "${REPL_CHECK[@]}"; do
 # 		# 	${NHCMD} rm -f ${FOLDER}/test.repl
@@ -459,7 +525,7 @@ STATUS(){
 # 			##REPLICATION CHECK
 # 			# nid=1
 # 			# for nip in "${NODE_HOSTS[@]}"; do
-# 			# 	if [ "${nip}" = "${PRIMARYHOST}" ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
+# 			# 	if [ "${nip}" = "${NODE_HOSTS[0]}" ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
 # 			# 	if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
 # 			# 		checkhost=$(CHECK_HOST ${nip})
 # 			# 		if [ "${checkhost}" != "false" ]; then
@@ -731,7 +797,7 @@ LISTCERTS(){
 	done
 	unset IFS
 	if [ ! -z ${LOCAL_SERVICES+x} ]; then
-		NCMD="ssh root@${PRIMARYHOST}"
+		NCMD="ssh root@${NODE_HOSTS[0]}"
 		${NCMD} rm -f /tmp/ssllist
 		${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist
 	else