diff --git a/defaults.inc b/defaults.inc index e8563699..66ceeb2e 100755 --- a/defaults.inc +++ b/defaults.inc @@ -9,6 +9,8 @@ FOLDER=/opt/idssys/nodemgmt SCRIPT=${FOLDER}/nodemgmt-scripts.sh RENOTIFY=60 +declare -i errtime + IFS=, NODE_HOSTS=(${NODEHOSTS}) NODE_SERVICES=(${NODESERVICES}) diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index e038a24a..5bf6601b 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -80,11 +80,12 @@ STATUS() { uptime=`${NCMD} uptime -p` echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}" DIVIDER false green - checkhost=$(CHECK_HOST ${nip}) + if [ "${NCMD}" != "" ]; then + checkhost=$(CHECK_HOST ${nip}) + fi if [ "${checkhost}" != "false" ]; then rm -f ${FOLDER}/${nip}.down - for srvc in "${NODE_SERVICES[@]}" - do + for srvc in "${NODE_SERVICES[@]}"; do c=0; cw=22; spc='' spc1=${cw}-${#NM_SERVICES[${srvc}]} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done @@ -92,9 +93,22 @@ STATUS() { if [ "$(${NCMD} systemctl is-active ${srvc})" = "active" ]; then echo -e "${idsCL[Green]}Running${idsCL[Default]}" rm -f ${FOLDER}/${nip}-${srvc}.down - else + rm -f ${FOLDER}/${nip}-${srvc}.errtime + elif [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" touch ${FOLDER}/${nip}-${srvc}.down + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + echo -e "${idsCL[Red]}Not Running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + touch ${FOLDER}/${nip}-${srvc}.down + else + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + fi fi done else @@ -191,6 +205,7 @@ STATUS-CHECK(){ elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} rm -f ${FOLDER}/localhost-${srvc}.down + rm -f ${FOLDER}/localhost-${srvc}.errtime fi done fi @@ -212,16 +227,20 @@ STATUS-CHECK(){ echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} touch ${FOLDER}/${nip}-${srvc}.down else - declare -i errtime errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - echo "errtime: ${errtime}" if [ $errtime -gt ${RENOTIFY} ]; then - echo "RENOTIFY" + if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} + touch ${FOLDER}/${nip}-${srvc}.down fi fi elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} rm -f ${FOLDER}/${nip}-${srvc}.down + rm -f ${FOLDER}/${nip}-${srvc}.errtime fi done elif [ ! -f ${FOLDER}/${nip}.down ]; then