From f86efd9a29d5eaddc9d9b5b64d003a13b29483ab Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Mon, 25 Mar 2019 12:27:33 -0500
Subject: [PATCH] Update nodemgmt-scripts.sh

---
 nodemgmt-scripts.sh | 106 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 105 insertions(+), 1 deletion(-)

diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index 3036a467..5d7a2d3f 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -38,6 +38,7 @@ STATUS(){
 		fi
 		# if [ ! -z ${LOCAL_SERVICES+x} ]; then
 			NHCMD="ssh root@${NODE_HOSTS[0]}"
+			LBHCMD="ssh root@${LB_HOSTS[0]}"
 			WHCMD="ssh root@${WEB_HOSTS[0]}"
 			# echo "HERE: '${NHCMD}' '${WHCMD}'"
 		# else
@@ -56,6 +57,19 @@ STATUS(){
 				fi
 			done
 		fi
+		if [ ! -z ${LB_REPL_CHECK+x} ]; then
+			for rcheck in "${LB_REPL_CHECK[@]}"; do
+				${LBHCMD} rm -f ${FOLDER}/test.repl
+				daterun=`date +%Y-%m-%d-%H-%M-%S`
+				if [ "${LBHCMD}" = "" ]; then
+					echo -e "LB-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
+					yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
+				else
+					${LBHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
+					yes | ${LBHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
+				fi
+			done
+		fi
 		for rcheck in "${WEB_REPL_CHECK[@]}"; do
 			${WHCMD} rm -f ${FOLDER}/test.repl
 			daterun=`date +%Y-%m-%d-%H-%M-%S`
@@ -225,6 +239,96 @@ STATUS(){
 		done
 		########################
 		nid=1
+		for nip in "${LB_HOSTS[@]}"; do
+			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
+				else NCMD="ssh root@${nip}"; LH=''
+			fi
+			if [ "${ST_ACTION}" != "check" ]; then
+				uptime=`${NCMD} uptime -p`
+				echo -e " ${idsST[Bold]}${idsCL[LightCyan]}LB-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+				DIVIDER false green
+			fi
+			if [ "${NCMD}" != "" ]; then
+				checkhost=$(CHECK_HOST ${nip})
+			fi
+			if [ "${checkhost}" != "false" ]; then
+				if [ -f ${FOLDER}/${nip}.down ]; then
+					if [ -f ${FOLDER}/${nip}.errtime ]; then
+						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+					else
+						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+					fi
+					rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
+					if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
+				fi
+				for srvc in "${LB_SERVICES_CHECK[@]}"; do
+					if [ "${ST_ACTION}" != "check" ]; then
+						c=0; cw=22; spc=''
+						spc1=${cw}-${#NM_SERVICES[${srvc}]}
+						until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+						echo -en " ${NM_SERVICES[${srvc}]}$spc: "
+					fi
+					
+					if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
+						if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
+							fi
+							if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
+							touch ${FOLDER}/${nip}-${srvc}.down
+						else
+							errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+							if [ $errtime -gt ${RENOTIFY} ]; then
+								if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+									mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
+								fi
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+								if [ "${ST_ACTION}" != "check" ]; then
+									echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+								fi
+								if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
+								touch ${FOLDER}/${nip}-${srvc}.down
+							fi
+						fi
+					else
+						if [ "${ST_ACTION}" != "check" ]; then
+							echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+						fi
+						if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
+							if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+							else
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+							fi
+							if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
+							rm -f ${FOLDER}/${nip}-${srvc}.down
+							rm -f ${FOLDER}/${nip}-${srvc}.errtime
+						fi
+					fi
+				done
+			else
+				errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+				if [ $errtime -gt ${RENOTIFY} ]; then
+					if [ ! -f ${FOLDER}/${nip}.errtime ]; then
+						mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
+					fi
+					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+					fi
+					if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
+				else
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
+					fi
+				fi
+				touch ${FOLDER}/${nip}.down
+			fi
+			if [ "${ST_ACTION}" != "check" ]; then echo; fi
+			nid=`expr $nid + 1`
+		done
+		########################
+		nid=1
 		for nip in "${WEB_HOSTS[@]}"; do
 			if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
 				else NCMD="ssh root@${nip}"; LH=''
@@ -1694,7 +1798,7 @@ NODEUPDATE() {
 		nid=`expr $nid + 1`
 	done
 	echo ""
-	echo -e "${idsCL[Green]}All Nodes and Webservers have been Updated.${idsCL[Default]}"
+	echo -e "${idsCL[Green]}All Nodes have been Updated.${idsCL[Default]}"
 	echo ""
 }