diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 346456a7..ab13ec58 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -95,6 +95,7 @@ STATUS(){ fi if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down + echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} else errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down) if [ $errtime -gt ${RENOTIFY} ]; then @@ -122,6 +123,8 @@ STATUS(){ if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime + echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE} + fi fi done @@ -180,7 +183,9 @@ STATUS(){ toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) fi rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime - # if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${EMAIL_NOTICE}; fi + if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${EMAIL_NOTICE}; fi + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} + fi if [ "${ST_ACTION}" != "dockers" ]; then @@ -218,13 +223,13 @@ STATUS(){ # echo "mysqlgo=${mysqlgo}" if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then - if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi touch ${FOLDER}/${nip}-${srvc}.down - # fi + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} + else errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) if [ $errtime -gt ${RENOTIFY} ]; then @@ -253,6 +258,7 @@ STATUS(){ toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) fi if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then rm -f ${FOLDER}/*-${srvc}.down @@ -300,6 +306,8 @@ STATUS(){ if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi rm -f ${FOLDER}/${nip}-${docker}.down rm -f ${FOLDER}/${nip}-${docker}.errtime + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE} + else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" @@ -307,6 +315,7 @@ STATUS(){ if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi touch ${FOLDER}/${nip}-${docker}.down + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE} else errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down) if [ $errtime -gt ${RENOTIFY} ]; then @@ -344,6 +353,7 @@ STATUS(){ echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node is down!${idsCL[Default]}" fi [ "${EMAIL_NOTICE}" != "" ] && echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE} + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE} else if [ "${ST_ACTION}" != "check" ]; then toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) @@ -457,6 +467,7 @@ STATUS(){ if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then touch ${FOLDER}/${nip}-${rcheck}.down if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\t$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} fi else if [ "${ST_ACTION}" != "check" ]; then @@ -465,6 +476,7 @@ STATUS(){ if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then rm -f ${FOLDER}/${nip}-${rcheck}.down if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\t$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} fi fi