diff --git a/defaults.inc b/defaults.inc index cbffaa02..bdcd82ab 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='5.1.134-08152024' +VERS='5.1.200-08222024' NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix ' @@ -116,6 +116,7 @@ NM_LOGFILE=${NM_LOGFOLDER}/logfile [ ! -d ${NM_LOGFOLDER} ] && mkdir ${NM_LOGFOLDER} [ "${NM_RENOTIFY_TIMEOUT}" == "" ] && NM_RENOTIFY_TIMEOUT=3600 +[ "${NM_ENABLE_RENOTIFY}" == "" ] && NM_ENABLE_RENOTIFY=1 NM_PDNS_WANDNS='wdns.scity.us' NM_PDNS_LANDNS='ldns.scity.us' diff --git a/inc/status.inc b/inc/status.inc index 5bff14ec..1d2c14a0 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -263,10 +263,12 @@ STATUS(){ if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] && ([[ "${nip}" != *"10.2."* ]] || [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host6.down ]) && ([ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)) -gt 60 ]); then [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE} - SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]}) -It has been down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)))" 1 - touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down + if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ]; then + SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]}) +It has been down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)))" 1 + touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent + fi fi fi @@ -616,8 +618,10 @@ STATUS_NODE(){ if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime touch ${NM_TMPFOLDER}/${nip}~${srvc}.down - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is down" 1 - touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent + if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is down" 1 + touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent + fi fi fi else @@ -724,9 +728,11 @@ STATUS_NODE(){ toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)) [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" touch ${NM_TMPFOLDER}/${nip}~${docker}.down - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_SERVICE_DESC[${docker}]} - is down" 1 - touch ${NM_TMPFOLDER}/${nip}~${docker}.sent - echo -en "${idsCL[Red]}Not Running, could not fix earlier, FIX MX!" + if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_SERVICE_DESC[${docker}]} - is down" 1 + touch ${NM_TMPFOLDER}/${nip}~${docker}.sent + fi + echo -en "${idsCL[Red]}Not Running, could not fix earlier, FIX ME!" fi else @@ -791,9 +797,9 @@ STATUS_NODE(){ echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" fi fi - if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then + if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then if [ "${checkhost}" != "false" ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSL has been down for $(SHOW_TIME ${toterrtime})" 1 + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSH has been down for $(SHOW_TIME ${toterrtime})" 1 else SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1 fi