diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 1cc39f30..5d42860e 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -27,6 +27,7 @@ STATUS(){ idsST=('') fi declare -i cw; declare -i spc1; declare -i c + ######################## if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}" @@ -51,7 +52,7 @@ STATUS(){ sleep 5 fi fi - + ######################## if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsST[Bold]}"; DIVIDER @@ -65,7 +66,6 @@ STATUS(){ echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" DIVIDER false green fi - for srvc in "${LOCAL_SERVICES[@]}"; do if [ "${ST_ACTION}" != "check" ]; then c=0; cw=22; spc='' @@ -73,20 +73,25 @@ STATUS(){ until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en " ${NM_SERVICES[${srvc}]}$spc: " fi - if [ "$(systemctl is-active ${srvc})" = "active" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - rm -f ${FOLDER}/localhost-${srvc}.down - else + if [ "$(systemctl is-active ${srvc})" != "active" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi - touch ${FOLDER}/localhost-${srvc}.down + if [ ! -f ${FOLDER}/localhost-${srvc}.down ]; then + echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} + touch ${FOLDER}/localhost-${srvc}.down + fi + elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi + echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} + rm -f ${FOLDER}/localhost-${srvc}.down; rm -f ${FOLDER}/localhost-${srvc}.errtime fi done echo fi + ######################## nid=1 for nip in "${NODE_HOSTS[@]}"; do if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' @@ -101,7 +106,10 @@ STATUS(){ checkhost=$(CHECK_HOST ${nip}) fi if [ "${checkhost}" != "false" ]; then - rm -f ${FOLDER}/${nip}.down + if [ -f ${FOLDER}/${nip}.down ]; then + rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime + echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL} + fi for srvc in "${NODE_SERVICES[@]}"; do if [ "${ST_ACTION}" != "check" ]; then c=0; cw=22; spc='' @@ -109,33 +117,35 @@ STATUS(){ until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en " ${NM_SERVICES[${srvc}]}$spc: " fi - if [ "$(${NCMD} systemctl is-active ${srvc})" = "active" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - rm -f ${FOLDER}/${nip}-${srvc}.down - rm -f ${FOLDER}/${nip}-${srvc}.errtime - elif [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - fi - touch ${FOLDER}/${nip}-${srvc}.down - else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime - fi - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - touch ${FOLDER}/${nip}-${srvc}.down - else + + if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi + echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} + touch ${FOLDER}/${nip}-${srvc}.down + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} + touch ${FOLDER}/${nip}-${srvc}.down + fi fi + elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi + echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} + rm -f ${FOLDER}/${nip}-${srvc}.down + rm -f ${FOLDER}/${nip}-${srvc}.errtime fi done else @@ -148,7 +158,7 @@ STATUS(){ nid=`expr $nid + 1` done fi - + ######################## if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsST[Bold]}"; DIVIDER @@ -165,7 +175,10 @@ STATUS(){ fi checkhost=$(CHECK_HOST ${nip}) if [ "${checkhost}" != "false" ]; then - rm -f ${FOLDER}/${nip}.down + if [ -f ${FOLDER}/${nip}.down ]; then + rm -f ${FOLDER}/${nip}.down + echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL} + fi for rcheck in "${REPL_CHECK[@]}"; do if [ "${ST_ACTION}" != "check" ]; then c=0; cw=22; spc='' @@ -186,23 +199,33 @@ STATUS(){ break fi done - if [ "${timeout}" != "true" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Good${idsCL[Default]}" - fi - rm -f ${FOLDER}/${nip}-${rcheck}.down - else + if [ "${timeout}" = "true" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" fi - touch ${FOLDER}/${nip}-${rcheck}.down + if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then + touch ${FOLDER}/${nip}-${rcheck}.down + echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL} + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Good${idsCL[Default]}" + fi + if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then + rm -f ${FOLDER}/${nip}-${rcheck}.down + echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL} + fi fi + done else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" fi - touch ${FOLDER}/${nip}.down + if [ ! -f ${FOLDER}/${nip}.down ]; then + touch ${FOLDER}/${nip}.down + echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL} + fi fi if [ "${ST_ACTION}" != "check" ]; then echo; fi fi