From 0c301eec2f76cc2d2ea058b782a1e45a284d49f5 Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Sat, 30 Nov 2024 12:00:01 -0600
Subject: [PATCH] update

---
 defaults.inc   |  2 +-
 inc/status.inc | 82 ++++++++++++++++++++++++++++----------------------
 2 files changed, 47 insertions(+), 37 deletions(-)

diff --git a/defaults.inc b/defaults.inc
index ed6ab48c..f6399add 100755
--- a/defaults.inc
+++ b/defaults.inc
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-VERS='5.3.14e-11292024'
+VERS='5.3.15-11302024'
 NM_BETA=false
 
 noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix '
diff --git a/inc/status.inc b/inc/status.inc
index e2165758..91a3937c 100755
--- a/inc/status.inc
+++ b/inc/status.inc
@@ -474,52 +474,61 @@ STATUS_NODE(){
 			else
 				NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"; LH=''
 			fi
-			if [ "${NCMD}" != "" ]; then
+			if [ "${NCMD}" != "" ] && [ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.* ]; then
 				checkhost=$(CHECK_HOST ${nip})
 				[ "${checkhost}" != "false" ] && checkhostssl=$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1)
+			elif [ -f ${STATUSRUN_TMPFOLDER}/.${nip}.up ]; then
+				checkhost=true
+				checkhostssl=ok
+			elif [ -f ${STATUSRUN_TMPFOLDER}/.${nip}.down ]; then
+				checkhost=false
+				checkhostssl=no
 			else
 				checkhost=true
 				checkhostssl=ok
 			fi
 			if [ "${checkhost}" != "false" ] && [ "${checkhostssl}" == "ok" ]; then
-				[ "${NODETYPE}" != "OFW" ] && cpu_usage=$(${NCMD} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage=""
-				if [ "${cpu_usage}" != "" ]; then
-					if [ "${NM_HOST_CPULEVELS[${nip}]}" != "" ]; then
-						cpu_warn=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 1)
-						cpu_crit=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 2)
-					else
-						cpu_warn=60
-						cpu_crit=80
-					fi
-					if [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_warn} ]; then
-						CUFC="${idsCL[Green]}"
-						if [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then
-							if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ]; then
-								echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.norm
-							elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]); then
-								[ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] NORMAL CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has returned to normal cpu usage: ${cpu_usage}%"
-								rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.*
+				if [ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.up ]; then
+					[ "${NODETYPE}" != "OFW" ] && cpu_usage=$(${NCMD} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage=""
+					if [ "${cpu_usage}" != "" ]; then
+						if [ "${NM_HOST_CPULEVELS[${nip}]}" != "" ]; then
+							cpu_warn=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 1)
+							cpu_crit=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 2)
+						else
+							cpu_warn=60
+							cpu_crit=80
+						fi
+						if [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_warn} ]; then
+							CUFC="${idsCL[Green]}"
+							if [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then
+								if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ]; then
+									echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.norm
+								elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]); then
+									[ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] NORMAL CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has returned to normal cpu usage: ${cpu_usage}%"
+									rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.*
+								fi
+							fi
+						elif [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_crit} ]; then
+							CUFC="${idsCL[LightYellow]}"
+							if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ]; then
+								echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn
+							elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then
+								SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] WARNING HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has high cpu usage: ${cpu_usage}%"
+								echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn
+								touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent
+							fi
+						else
+							CUFC="${idsCL[LightRed]}"
+							if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then
+								echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit
+							elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then
+								SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] CRITICAL HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has CRITICALLY high cpu usage: ${cpu_usage}%" 1
+								echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit
+								touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent
 							fi
 						fi
-					elif [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_crit} ]; then
-						CUFC="${idsCL[LightYellow]}"
-						if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ]; then
-							echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn
-						elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then
-							SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] WARNING HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has high cpu usage: ${cpu_usage}%"
-							echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn
-							touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent
-						fi
-					else
-						CUFC="${idsCL[LightRed]}"
-						if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then
-							echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit
-						elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then
-							SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] CRITICAL HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has CRITICALLY high cpu usage: ${cpu_usage}%" 1
-							echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit
-							touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent
-						fi
 					fi
+					touch ${STATUSRUN_TMPFOLDER}/.${nip}.up
 				fi
 				if [ "${STATUS_ACTION}" != "check" ]; then
 					if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then
@@ -773,6 +782,7 @@ STATUS_NODE(){
 			
 			else
 				[ "${STATUS_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}"
+				[ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.down ] && touch ${STATUSRUN_TMPFOLDER}/.${nip}.down
 				rm -f ${NM_TMPFOLDER}/${nip}~*
 				if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
 					if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then