From f92ece762ababa11f5bb3f35dd3278eb89e4cd05 Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Tue, 26 Feb 2019 23:00:08 -0600
Subject: [PATCH] Update nodemgmt-scripts.sh

---
 nodemgmt-scripts.sh | 92 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 91 insertions(+), 1 deletion(-)

diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index 9b26fc4a..36112d46 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -207,6 +207,96 @@ STATUS(){
 			if [ "${ST_ACTION}" != "check" ]; then echo; fi
 			nid=`expr $nid + 1`
 		done
+		########################
+		nid=1
+		for nip in "${WEB_HOSTS[@]}"; do
+			if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
+				else NCMD="ssh root@${nip}"; LH=''
+			fi
+			if [ "${ST_ACTION}" != "check" ]; then
+				uptime=`${NCMD} uptime -p`
+				echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Webserver${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+				DIVIDER false green
+			fi
+			if [ "${NCMD}" != "" ]; then
+				checkhost=$(CHECK_HOST ${nip})
+			fi
+			if [ "${checkhost}" != "false" ]; then
+				if [ -f ${FOLDER}/${nip}.down ]; then
+					if [ -f ${FOLDER}/${nip}.errtime ]; then
+						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+					else
+						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+					fi
+					rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
+					echo "iDS-MySQL${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Webserver${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
+				fi
+				for srvc in "${WEB_SERVICES_CHECK[@]}"; do
+					if [ "${ST_ACTION}" != "check" ]; then
+						c=0; cw=22; spc=''
+						spc1=${cw}-${#NM_SERVICES[${srvc}]}
+						until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+						echo -en " ${NM_SERVICES[${srvc}]}$spc: "
+					fi
+					
+					if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
+						if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
+							fi
+							echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Webserver${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+							touch ${FOLDER}/${nip}-${srvc}.down
+						else
+							errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+							if [ $errtime -gt ${RENOTIFY} ]; then
+								if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+									mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
+								fi
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+								if [ "${ST_ACTION}" != "check" ]; then
+									echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+								fi
+								echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Webserver${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+								touch ${FOLDER}/${nip}-${srvc}.down
+							fi
+						fi
+					else
+						if [ "${ST_ACTION}" != "check" ]; then
+							echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+						fi
+						if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
+							if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+							else
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+							fi
+							echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Webserver${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+							rm -f ${FOLDER}/${nip}-${srvc}.down
+							rm -f ${FOLDER}/${nip}-${srvc}.errtime
+						fi
+					fi
+				done
+			else
+				errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+				if [ $errtime -gt ${RENOTIFY} ]; then
+					if [ ! -f ${FOLDER}/${nip}.errtime ]; then
+						mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
+					fi
+					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+					fi
+					echo "iDS-Webserver${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Webserver${nid}-${nip}" ${STATUS_CHECK_EMAIL}
+				else
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
+					fi
+				fi
+				touch ${FOLDER}/${nip}.down
+			fi
+			if [ "${ST_ACTION}" != "check" ]; then echo; fi
+			nid=`expr $nid + 1`
+		done
 	fi
 	########################
 	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then
@@ -1179,7 +1269,7 @@ SET-PERMISSIONS(){
         	echo -en "${idsCL[Cyan]}Gitea Folder         ${idsCL[Default]}"
 			${NCMD} chown -R git:git /var/lib/gitea
 			${NCMD} chmod -R 750 /var/lib/gitea
-			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
+			echo -e "${idsCL[Green]}Complete${idsCL[Def ault]}"
 			echo ""
 		fi
 		nid=`expr $nid + 1`