diff --git a/defaults.inc b/defaults.inc index fadaefd6..cf5b4a80 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='5.1.56a-01062024' -NM_BETA=true +VERS='5.1.57-01062024' +NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck ' CERT_DAEMON='/snap/bin/certbot' diff --git a/inc/status.inc b/inc/status.inc index dbbc909a..e9b17f67 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -33,8 +33,7 @@ STATUS(){ ## REPLICATION CHECK SETUP ############################ # REPLRUN=0 - echo "WHAT: $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replcheck)) -ge $(expr ${NM_REPL_CHECK_TIMEOUT} * 60)" - [ "${NM_DISABLE_REPL_CHECK}" != "true" ] && ([ ! -f ${NM_TMPFOLDER}/.replcheck ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replcheck)) -ge $(expr ${NM_REPL_CHECK_TIMEOUT} * 60) ]) && REPLRUN=1 || REPLRUN=0 + [ "${NM_DISABLE_REPL_CHECK}" != "true" ] && ([ ! -f ${NM_TMPFOLDER}/.replcheck ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replcheck)) -ge $(expr ${NM_REPL_CHECK_TIMEOUT} \* 60) ]) && REPLRUN=1 || REPLRUN=0 [ ${REPLRUN} -eq 1 ] && touch ${NM_TMPFOLDER}/.replcheck # if [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "" ]; then if [ "${STATUS_ACTION}" == "repl" ] || ([ "${STATUS_ACTION}" == "" ] && [ "${ntss}" == "" ]) || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && [ "${NM_REPL_CHECK[${ntss}]}" != "" ] && ([ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "repl" ])); then @@ -443,7 +442,37 @@ STATUS_NODE(){ fi if [ "${checkhost}" != "false" ] && [ "${checkhostssl}" == "ok" ]; then [ "${NODETYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage="" - + if [ "${cpu_usage}" != "" ]; then + if [ $(ROUND_NUMBER ${cpu_usage}) < 1 ]; then + CUFC="${idsCL[Green]}" + if [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ]; then + if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ]; then + touch ${NM_TMPFOLDER}/${nip}.cpu_usage.norm + elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 180 ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] NORMAL CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has returned to normal cpu usage: ${cpu_usage}%" + rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.* + fi + fi + elif [ $(ROUND_NUMBER ${cpu_usage}) < 80 ]; then + CUFC="${idsCL[LightYellow]}" + [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] && rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm + if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ]; then + touch ${NM_TMPFOLDER}/${nip}.cpu_usage.warn + elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 180 ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] WARNING HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has high cpu usage: ${cpu_usage}%" + touch ${NM_TMPFOLDER}/${nip}.cpu_usage.warn + fi + else + CUFC="${idsCL[LightRed]}" + [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] && rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm + if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then + touch ${NM_TMPFOLDER}/${nip}.cpu_usage.crit + elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 60 ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] CRITICAL HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has CRITICALLY high cpu usage: ${cpu_usage}%" 1 + touch ${NM_TMPFOLDER}/${nip}.cpu_usage.crit + fi + fi + fi if [ "${STATUS_ACTION}" != "check" ]; then if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then if [ "${NODETYPE}" == "OFW" ]; then @@ -459,7 +488,7 @@ STATUS_NODE(){ else uptime=$(${NCMD} uptime -p) fi - [ "${STATUS_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${CUFC}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}" else echo -e "${idsCL[Default]}" fi @@ -475,36 +504,6 @@ STATUS_NODE(){ rm -f ${NM_TMPFOLDER}/${nip}.* fi - if [ "${cpu_usage}" != "" ]; then - echo "HERE: $(ROUND_NUMBER ${cpu_usage})" - if [ $(ROUND_NUMBER ${cpu_usage}) < 1 ]; then - if [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ]; then - if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ]; then - touch ${NM_TMPFOLDER}/${nip}.cpu_usage.norm - elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 180 ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] NORMAL CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has returned to normal cpu usage: ${cpu_usage}%" - rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.* - fi - fi - elif [ $(ROUND_NUMBER ${cpu_usage}) < 80 ]; then - [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] && rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm - if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ]; then - touch ${NM_TMPFOLDER}/${nip}.cpu_usage.warn - elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 180 ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] WARNING HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has high cpu usage: ${cpu_usage}%" - touch ${NM_TMPFOLDER}/${nip}.cpu_usage.warn - fi - else - [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] && rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm - if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then - touch ${NM_TMPFOLDER}/${nip}.cpu_usage.crit - elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 60 ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] CRITICAL HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has CRITICALLY high cpu usage: ${cpu_usage}%" 1 - touch ${NM_TMPFOLDER}/${nip}.cpu_usage.crit - fi - fi - fi - ######################## ## SERVICES CHECK ########################