diff --git a/inc/status.inc b/inc/status.inc index 92dde8b2..11035863 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -530,46 +530,48 @@ STATUS_NODE(){ fi if [ "${checkhost}" != "false" ] && [ "${checkhostssh}" == "ok" ]; then if [ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.up ]; then + ############## ALERTING ON HIGH CPU USAGE ############## - # [ "${NODETYPE}" != "OFW" ] && cpu_usage=$(${NCMD} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage="" - # if [ "${cpu_usage}" != "" ]; then - # if [ "${NM_HOST_CPULEVELS[${nip}]}" != "" ]; then - # cpu_warn=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 1) - # cpu_crit=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 2) - # else - # cpu_warn=60 - # cpu_crit=80 - # fi - # if [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_warn} ]; then - # CUFC="${idsCL[Green]}" - # if [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then - # if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ]; then - # echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.norm - # elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]); then - # [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] NORMAL CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has returned to normal cpu usage: ${cpu_usage}%" - # rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.* - # fi - # fi - # elif [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_crit} ]; then - # CUFC="${idsCL[LightYellow]}" - # if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ]; then - # echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn - # elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] WARNING HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has high cpu usage: ${cpu_usage}%" - # echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn - # touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent - # fi - # else - # CUFC="${idsCL[LightRed]}" - # if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then - # echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit - # elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] CRITICAL HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has CRITICALLY high cpu usage: ${cpu_usage}%" 1 - # echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit - # touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent - # fi - # fi - # fi + [ "${NODETYPE}" != "OFW" ] && cpu_usage=$(${NCMD} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage="" + if [ "${cpu_usage}" != "" ]; then + if [ "${NM_HOST_CPULEVELS[${nip}]}" != "" ]; then + cpu_warn=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 1) + cpu_crit=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 2) + else + cpu_warn=60 + cpu_crit=80 + fi + if [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_warn} ]; then + CUFC="${idsCL[Green]}" + if [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then + if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ]; then + echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.norm + elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]); then + [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] NORMAL CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has returned to normal cpu usage: ${cpu_usage}%" + rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.* + fi + fi + elif [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_crit} ]; then + CUFC="${idsCL[LightYellow]}" + if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ]; then + echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn + elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] WARNING HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has high cpu usage: ${cpu_usage}%" + echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn + touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent + fi + else + CUFC="${idsCL[LightRed]}" + if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then + echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit + elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] CRITICAL HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has CRITICALLY high cpu usage: ${cpu_usage}%" 1 + echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit + touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent + fi + fi + fi + touch ${STATUSRUN_TMPFOLDER}/.${nip}.up fi if [ "${STATUS_ACTION}" != "check" ]; then