update

defaults.inc

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-VERS='5.1.134-08152024'
+VERS='5.1.200-08222024'
 NM_BETA=false
 
 noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix '
@@ -116,6 +116,7 @@ NM_LOGFILE=${NM_LOGFOLDER}/logfile
 [ ! -d ${NM_LOGFOLDER} ] && mkdir ${NM_LOGFOLDER}
 
 [ "${NM_RENOTIFY_TIMEOUT}" == "" ] && NM_RENOTIFY_TIMEOUT=3600
+[ "${NM_ENABLE_RENOTIFY}" == "" ] && NM_ENABLE_RENOTIFY=1
 
 NM_PDNS_WANDNS='wdns.scity.us'
 NM_PDNS_LANDNS='ldns.scity.us'

inc/status.inc

@@ -263,10 +263,12 @@ STATUS(){
 										if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] && ([[ "${nip}" != *"10.2."* ]] || [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host6.down ]) && ([ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)) -gt 60 ]); then
 											[ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime
 											echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
-											SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
-It has been down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)))" 1
-											touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent
 											touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
+											if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ]; then
+												SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
+It has been down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)))" 1
+												touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent
+											fi
 										fi
 									fi
 										
@@ -616,8 +618,10 @@ STATUS_NODE(){
 									if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then
 										[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
 										touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
-										SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is down" 1
+										if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then
-										touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent
+											SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is down" 1
+											touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent
+										fi
 									fi
 								fi
 							else
@@ -724,9 +728,11 @@ STATUS_NODE(){
 										toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
 										[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
 										touch ${NM_TMPFOLDER}/${nip}~${docker}.down
-										SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_SERVICE_DESC[${docker}]} - is down" 1
+										if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ]; then
-										touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
+											SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_SERVICE_DESC[${docker}]} - is down" 1
-										echo -en "${idsCL[Red]}Not Running, could not fix earlier, FIX MX!"
+											touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
+										fi
+										echo -en "${idsCL[Red]}Not Running, could not fix earlier, FIX ME!"
 									fi
 									
 								else
@@ -791,9 +797,9 @@ STATUS_NODE(){
 							echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
 						fi
 					fi
-					if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then
+					if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then
 						if [ "${checkhost}" != "false" ]; then
-							SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSL has been down for $(SHOW_TIME ${toterrtime})" 1
+							SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSH has been down for $(SHOW_TIME ${toterrtime})" 1
 						else
 							SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
 						fi