From a48d90fd325be8ce0de45e7b867feb6c606e923f Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Fri, 8 Feb 2019 00:32:43 -0600
Subject: [PATCH] Update nodemgmt-scripts.sh

---
 nodemgmt-scripts.sh | 56 +++++++++++++++++++++++++++++++++++----------
 1 file changed, 44 insertions(+), 12 deletions(-)

diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index 6c954b33..c19237a6 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -73,23 +73,46 @@ STATUS(){
 					until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
 					echo -en " ${NM_SERVICES[${srvc}]}$spc: "
 				fi
+				
 				if [ "$(systemctl is-active ${srvc})" != "active" ]; then
-					if [ "${ST_ACTION}" != "check" ]; then
-						echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
+					if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
+						if [ "${ST_ACTION}" != "check" ]; then
+							echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
+						fi
+						echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}
+						touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
+					else
+						errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
+						if [ $errtime -gt ${RENOTIFY} ]; then
+							if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
+								mv ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
+							fi
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+							fi
+							echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}
+							touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
+						fi
 					fi
-					if [ ! -f ${FOLDER}/localhost-${srvc}.down ]; then
-						echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
-						touch ${FOLDER}/localhost-${srvc}.down
-					fi
-				elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then
+				else
 					if [ "${ST_ACTION}" != "check" ]; then
 						echo -e "${idsCL[Green]}Running${idsCL[Default]}"
 					fi
-					echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
-					rm -f ${FOLDER}/localhost-${srvc}.down; rm -f ${FOLDER}/localhost-${srvc}.errtime
+					if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
+						if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
+						else
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
+						fi
+						echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}
+						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
+						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
+					fi
 				fi
 			done
-			echo
+			
+			if [ "${ST_ACTION}" != "check" ]; then echo; fi
 		fi
 		########################
 		nid=1
@@ -107,8 +130,13 @@ STATUS(){
 			fi
 			if [ "${checkhost}" != "false" ]; then
 				if [ -f ${FOLDER}/${nip}.down ]; then
+					if [ -f ${FOLDER}/${nip}.errtime ]; then
+						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+					else
+						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+					fi
 					rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
-					echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
+					echo "iDS-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
 				fi
 				for srvc in "${NODE_SERVICES[@]}"; do
 					if [ "${ST_ACTION}" != "check" ]; then
@@ -144,7 +172,11 @@ STATUS(){
 							echo -e "${idsCL[Green]}Running${idsCL[Default]}"
 						fi
 						if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
-							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+							if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+							else
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+							fi
 							echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
 							rm -f ${FOLDER}/${nip}-${srvc}.down
 							rm -f ${FOLDER}/${nip}-${srvc}.errtime