diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 5d40bd5e..6e32adc6 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -245,7 +245,7 @@ STATUS(){ toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) fi rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Webserver-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi fi for srvc in "${WEB_SERVICES_CHECK[@]}"; do if [ "${ST_ACTION}" != "check" ]; then @@ -313,6 +313,95 @@ STATUS(){ if [ "${ST_ACTION}" != "check" ]; then echo; fi nid=`expr $nid + 1` done + nid=1 + for nip in "${LB_HOSTS[@]}"; do + if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' + else NCMD="ssh root@${nip}"; LH='' + fi + if [ "${ST_ACTION}" != "check" ]; then + uptime=`${NCMD} uptime -p` + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}LB-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" + DIVIDER false green + fi + if [ "${NCMD}" != "" ]; then + checkhost=$(CHECK_HOST ${nip}) + fi + if [ "${checkhost}" != "false" ]; then + if [ -f ${FOLDER}/${nip}.down ]; then + if [ -f ${FOLDER}/${nip}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + else + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + fi + rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi + fi + for srvc in "${LB_SERVICES_CHECK[@]}"; do + if [ "${ST_ACTION}" != "check" ]; then + c=0; cw=22; spc='' + spc1=${cw}-${#NM_SERVICES[${srvc}]} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${NM_SERVICES[${srvc}]}$spc: " + fi + + if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + fi + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + touch ${FOLDER}/${nip}-${srvc}.down + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + touch ${FOLDER}/${nip}-${srvc}.down + fi + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi + if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then + if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + else + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + fi + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + rm -f ${FOLDER}/${nip}-${srvc}.down + rm -f ${FOLDER}/${nip}-${srvc}.errtime + fi + fi + done + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${nip}.errtime ]; then + mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" + fi + fi + touch ${FOLDER}/${nip}.down + fi + if [ "${ST_ACTION}" != "check" ]; then echo; fi + nid=`expr $nid + 1` + done fi ######################## if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then