Update nodemgmt-scripts.sh

nodemgmt-scripts.sh

@@ -499,6 +499,93 @@ STATUS(){
 	echo "runtime: ${runtime}"
 }
 
+CHECK_DOCKER_SERVICES(){
+	
+	for NTYPE in "${NODE_TYPES[@]}"; do
+		dockers=${NTYPE}_DOCKER[@]
+		hosts=${NTYPE}_HOSTS[@]
+		
+		if [ "${dockers}" != "" ]; then
+			if [ "${ST_ACTION}" != "check" ]; then
+				echo -e "${idsST[Bold]}"; DIVIDER
+				echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
+				DIVIDER; echo -e "${idsST[Reset]}"
+			fi
+			nid=1
+		
+			for nip in "${!hosts}"; do
+				if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
+					else NCMD="ssh root@${nip}"; LH=''
+				fi
+				if [ "${NCMD}" != "" ]; then
+					checkhost=$(CHECK_HOST ${nip})
+				fi
+				if [ "${checkhost}" != "false" ]; then
+					if [ "${ST_ACTION}" != "check" ]; then
+						uptime=`${NCMD} uptime -p`
+						echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+						DIVIDER false green
+					fi
+					if [ -f ${FOLDER}/${nip}.down ]; then
+						if [ -f ${FOLDER}/${nip}.errtime ]; then
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+						else
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+						fi
+						rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
+						# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
+					fi
+
+					for docker in "${!dockers}"; do
+						if [ "${ST_ACTION}" != "check" ]; then
+							c=0; cw=30; spc=''
+							spc1=${cw}-${#NM_DOCKERS[${docker}]}
+							until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+							echo -en " ${NM_DOCKERS[${docker}]}$spc: "
+						fi
+						if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
+						    if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
+						        ${NCMD} docker rm ${docker}
+						    fi
+						    ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up
+							sleep 10s
+							if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
+								if [ "${ST_ACTION}" != "check" ]; then
+									echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
+								fi
+							else
+								if [ "${ST_ACTION}" != "check" ]; then
+									echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
+								fi
+							fi
+						else
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+							fi
+						fi
+					done
+				else
+					if [ ! -f ${FOLDER}/${nip}.down ]; then
+						touch ${FOLDER}/${nip}.down
+						if [ ! -f ${FOLDER}/${nip}.errtime ]; then
+							touch ${FOLDER}/${nip}.errtime
+						fi
+					else
+						errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+					fi
+					if [ "${ST_ACTION}" != "check" ]; then
+						toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+						echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+					fi
+				fi
+				if [ "${ST_ACTION}" != "check" ]; then echo; fi
+				nid=`expr $nid + 1`
+			done
+		done
+	fi
+
+}
+
 NEWCERT(){
 	echo ""
 	if [ -z ${1+x} ]; then