From caff93d0f2b95ee80c7cc48996adb592dc7aea57 Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Sat, 17 Dec 2022 11:14:48 -0600
Subject: [PATCH] Update nodemgmt-scripts.sh

---
 nodemgmt-scripts.sh | 82 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)

diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index 8004c3e2..b02daa1f 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -265,6 +265,88 @@ STATUS(){
 				if [ "${ST_ACTION}" != "check" ]; then echo; fi
 				nid=`expr $nid + 1`
 			done
+			
+			######### CHECK DOCKER STATUS ###############
+			dockers=${NTYPE}_DOCKER[@]
+			hosts=${NTYPE}_HOSTS[@]
+		
+			if [[ -v ${NTYPE}_DOCKER ]]; then
+				if [ "${ST_ACTION}" != "check" ]; then
+					echo -e "${idsST[Bold]}"; DIVIDER
+					echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
+					DIVIDER; echo -e "${idsST[Reset]}"
+				fi
+				nid=1
+		
+				for nip in "${!hosts}"; do
+					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
+						else NCMD="ssh root@${nip}"; LH=''
+					fi
+					if [ "${NCMD}" != "" ]; then
+						checkhost=$(CHECK_HOST ${nip})
+					fi
+					if [ "${checkhost}" != "false" ]; then
+						if [ "${ST_ACTION}" != "check" ]; then
+							uptime=`${NCMD} uptime -p`
+							echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+							DIVIDER false green
+						fi
+						if [ -f ${FOLDER}/${nip}.down ]; then
+							if [ -f ${FOLDER}/${nip}.errtime ]; then
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+							else
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+							fi
+							rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
+						fi
+
+						for docker in "${!dockers}"; do
+							if [ "${ST_ACTION}" != "check" ]; then
+								c=0; cw=30; spc=''
+								spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
+								until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+								echo -en " ${NM_DOCKERS[${docker}]}$spc: "
+							fi
+							if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
+							    if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
+							        ${NCMD} docker rm ${docker}
+							    fi
+							    ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d
+								sleep 10s
+								if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
+									if [ "${ST_ACTION}" != "check" ]; then
+										echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
+									fi
+								else
+									if [ "${ST_ACTION}" != "check" ]; then
+										echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
+									fi
+								fi
+							else
+								if [ "${ST_ACTION}" != "check" ]; then
+									echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+								fi
+							fi
+						done
+					else
+						if [ ! -f ${FOLDER}/${nip}.down ]; then
+							touch ${FOLDER}/${nip}.down
+							if [ ! -f ${FOLDER}/${nip}.errtime ]; then
+								touch ${FOLDER}/${nip}.errtime
+							fi
+						else
+							errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+						fi
+						if [ "${ST_ACTION}" != "check" ]; then
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+							echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+						fi
+					fi
+					if [ "${ST_ACTION}" != "check" ]; then echo; fi
+					nid=`expr $nid + 1`
+				done
+			fi
+			
 		done
 		
 	fi