diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index d6966107..45d08d75 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -156,77 +156,79 @@ STATUS(){ fi var2=${NTYPE}_SERVICES_CHECK[@] for srvc in "${!var2}"; do - if [ "${ST_ACTION}" != "check" ]; then - c=0; cw=22; spc='' - spc1=${cw}-${#NM_SERVICES[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${NM_SERVICES[${srvc}]}$spc: " - fi + if [ "${NIP}" != "10.5.10.122" ] && [ ${srvc} != "gitea" ]; then + if [ "${ST_ACTION}" != "check" ]; then + c=0; cw=22; spc='' + spc1=${cw}-${#NM_SERVICES[${srvc}]} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${NM_SERVICES[${srvc}]}$spc: " + fi - if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then - if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then - if [ "${srvc}" == "gitea" ]; then - if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then - SERVICE gitea stop q - SET-PERMISSIONS gitea q - SERVICE gitea start q - fi - if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running - could not fix${idsCL[Default]}" - fi - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down. Could not fix." | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - touch ${FOLDER}/${nip}-${srvc}.down - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running - FIXED${idsCL[Default]}" - fi - if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then - if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - else - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - fi - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! (!!FIXED!!) It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - rm -f ${FOLDER}/${nip}-${srvc}.down - rm -f ${FOLDER}/${nip}-${srvc}.errtime - fi - fi - else + if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then + # if [ "${srvc}" == "gitea" ]; then + # if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then + # SERVICE gitea stop q + # SET-PERMISSIONS gitea q + # SERVICE gitea start q + # fi + # if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then + # if [ "${ST_ACTION}" != "check" ]; then + # echo -e "${idsCL[Red]}Not Running - could not fix${idsCL[Default]}" + # fi + # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down. Could not fix." | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + # touch ${FOLDER}/${nip}-${srvc}.down + # else + # if [ "${ST_ACTION}" != "check" ]; then + # echo -e "${idsCL[Green]}Running - FIXED${idsCL[Default]}" + # fi + # if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then + # if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + # else + # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + # fi + # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! (!!FIXED!!) It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + # rm -f ${FOLDER}/${nip}-${srvc}.down + # rm -f ${FOLDER}/${nip}-${srvc}.errtime + # fi + # fi + # else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + fi + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + touch ${FOLDER}/${nip}-${srvc}.down + # fi + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + touch ${FOLDER}/${nip}-${srvc}.down fi - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - touch ${FOLDER}/${nip}-${srvc}.down fi else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime - fi - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - touch ${FOLDER}/${nip}-${srvc}.down + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then - if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - else - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then + if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + else + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + fi + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + rm -f ${FOLDER}/${nip}-${srvc}.down + rm -f ${FOLDER}/${nip}-${srvc}.errtime fi - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - rm -f ${FOLDER}/${nip}-${srvc}.down - rm -f ${FOLDER}/${nip}-${srvc}.errtime fi fi done