diff --git a/defaults.inc b/defaults.inc index 16607705..c0ce036d 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='4.6.23-04302023' +VERS='4.6.24-04302023' #NODETYPES=WEB,LB NODETYPES=MYSQL,LB,WEB,NC,WPD,LPD,MM,PW diff --git a/inc/status.inc b/inc/status.inc index 4e7805e7..e190b204 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -150,11 +150,11 @@ STATUS(){ DIVIDER false green fi - if [ -f ${FOLDER}/${nip}.down ]; then - if [ -f ${FOLDER}/${nip}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + if [ -f ${TMPFOLDER}/${nip}.down ]; then + if [ -f ${TMPFOLDER}/${nip}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) else - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) fi rm -f ${FOLDER}/${nip}.* SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" @@ -185,25 +185,25 @@ STATUS(){ fi # echo "mysqlgo=${mysqlgo}" if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then + if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 - touch ${FOLDER}/${nip}-${srvc}.down + touch ${TMPFOLDER}/${nip}-${srvc}.down echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) if [ $errtime -gt ${RENOTIFY} ]; then - [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ] && mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - touch ${FOLDER}/${nip}-${srvc}.down + [ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}-${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime + toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) + touch ${TMPFOLDER}/${nip}-${srvc}.down # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & fi [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" fi else [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" - if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then - [ -f ${FOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + if [ -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then + [ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} @@ -250,22 +250,22 @@ STATUS(){ # fi # # echo "mysqlgo=${mysqlgo}" # if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - # if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then + # if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then # if [ "${ST_ACTION}" != "check" ]; then # echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" # fi # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 - # touch ${FOLDER}/${nip}-${srvc}.down + # touch ${TMPFOLDER}/${nip}-${srvc}.down # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} # # else - # errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + # errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) # if [ $errtime -gt ${RENOTIFY} ]; then - # if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - # mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + # if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then + # mv ${TMPFOLDER}/${nip}-${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime # fi - # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - # touch ${FOLDER}/${nip}-${srvc}.down + # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) + # touch ${TMPFOLDER}/${nip}-${srvc}.down # if [ "${srvc}" = "mysql" ]; then # ${NCMD} systemctl restart ${srvc} # fi @@ -278,17 +278,17 @@ STATUS(){ # if [ "${ST_ACTION}" != "check" ]; then # echo -e "${idsCL[Green]}Running${idsCL[Default]}" # fi - # if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then - # if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + # if [ -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then + # if [ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then + # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) # else - # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) # fi # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} # - # rm -f ${FOLDER}/${nip}-${srvc}.down - # rm -f ${FOLDER}/${nip}-${srvc}.errtime + # rm -f ${TMPFOLDER}/${nip}-${srvc}.down + # rm -f ${TMPFOLDER}/${nip}-${srvc}.errtime # # fi # # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then @@ -371,23 +371,23 @@ STATUS(){ fi else - if [ ! -f ${FOLDER}/${nip}.down ]; then - touch ${FOLDER}/${nip}.down - if [ ! -f ${FOLDER}/${nip}.errtime ]; then - touch ${FOLDER}/${nip}.errtime + if [ ! -f ${TMPFOLDER}/${nip}.down ]; then + touch ${TMPFOLDER}/${nip}.down + if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then + touch ${TMPFOLDER}/${nip}.errtime fi if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" fi SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1 echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE} else if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" fi - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) # if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE} # fi @@ -422,7 +422,7 @@ STATUS(){ var=${NTYPE}_HOSTS[@] for nip in "${!var}"; do if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then - if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then + if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then var2=${NTYPE}_REPL_CHECK[@] for rcheck in "${!var2}"; do REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1 @@ -446,7 +446,7 @@ STATUS(){ echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" DIVIDER false green fi - if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then + if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then var2=${NTYPE}_REPL_CHECK[@] for rcheck in "${!var2}"; do if [ "${ST_ACTION}" != "check" ]; then @@ -488,7 +488,7 @@ STATUS(){ else if [ "${ST_ACTION}" != "check" ]; then [ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}" - [ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}" + [ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}" fi fi if [ "${ST_ACTION}" != "check" ]; then echo; fi diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 2269543b..28ba6fee 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -40,14 +40,14 @@ CHECK_DOCKER_SERVICES(){ echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" DIVIDER false green fi - if [ -f ${FOLDER}/${nip}.down ]; then - if [ -f ${FOLDER}/${nip}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + if [ -f ${TMPFOLDER}/${nip}.down ]; then + if [ -f ${TMPFOLDER}/${nip}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) else - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) fi - rm -f ${FOLDER}/${nip}.down - rm -f ${FOLDER}/${nip}.errtime + rm -f ${TMPFOLDER}/${nip}.down + rm -f ${TMPFOLDER}/${nip}.errtime SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" fi @@ -80,16 +80,16 @@ CHECK_DOCKER_SERVICES(){ fi done else - if [ ! -f ${FOLDER}/${nip}.down ]; then - touch ${FOLDER}/${nip}.down - if [ ! -f ${FOLDER}/${nip}.errtime ]; then - touch ${FOLDER}/${nip}.errtime + if [ ! -f ${TMPFOLDER}/${nip}.down ]; then + touch ${TMPFOLDER}/${nip}.down + if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then + touch ${TMPFOLDER}/${nip}.errtime fi else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) fi if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" fi fi