From aa3d9b11b493398dd43e9ff8ac101fd473ed194b Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Wed, 6 Dec 2023 17:39:27 -0600 Subject: [PATCH] Update nodemgmt-scripts.sh --- nodemgmt-scripts.sh | 73 +++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 01e38379..b5ff5c5e 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -879,38 +879,59 @@ GUI(){ if [ "${2}" == "freespace" ]; then STATUS freespace ${3} else - STATUS_START=$(date +%Y-%m-%d-%H-%M-%S) - STATUSRUN_TMPFOLDER="${NM_TMPFOLDER}/status-${STATUS_START}" - mkdir ${STATUSRUN_TMPFOLDER} - touch "${STATUSRUN_TMPFOLDER}/status-check.running" + STATUSRUNS=($(find ${NM_TMPFOLDER}/* -maxdepth 1 -type d -iname "status-*" 2>/dev/null)) + if [ ${STATUSRUNS} -le 3 ]; then + STATUS_START=$(date +%Y-%m-%d-%H-%M-%S) + STATUSRUN_TMPFOLDER="${NM_TMPFOLDER}/status-${STATUS_START}" + mkdir ${STATUSRUN_TMPFOLDER} + touch "${STATUSRUN_TMPFOLDER}/status-check.running" + rm -f ${NM_TMPFOLDER}/.overload - if [ "${2}" == "report" ]; then - echo "${STATUS_START} - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE} - echo -e "${STATUS_START} - Starting status check #############################\n" > ${STATUSRUN_TMPFOLDER}/status-check.running - STATUS report ${3} >> ${STATUSRUN_TMPFOLDER}/status-check.running - echo -e "\n\n$(date +%Y-%m-%d-%H-%M-%S) - Finished status check #############################\n" >> ${STATUSRUN_TMPFOLDER}/status-check.running - echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE} + if [ "${2}" == "report" ]; then + echo "${STATUS_START} - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE} + echo -e "${STATUS_START} - Starting status check #############################\n" > ${STATUSRUN_TMPFOLDER}/status-check.running + STATUS report ${3} ${4} >> ${STATUSRUN_TMPFOLDER}/status-check.running + echo -e "\n\n$(date +%Y-%m-%d-%H-%M-%S) - Finished status check #############################\n" >> ${STATUSRUN_TMPFOLDER}/status-check.running + echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE} - rm -f ${NM_LOGFOLDER}/status-check.lastrun - mv ${STATUSRUN_TMPFOLDER}/status-check.running ${NM_LOGFOLDER}/status-check.lastrun - rm -Rf ${STATUSRUN_TMPFOLDER} + rm -f ${NM_LOGFOLDER}/status-check.lastrun + mv ${STATUSRUN_TMPFOLDER}/status-check.running ${NM_LOGFOLDER}/status-check.lastrun + rm -Rf ${STATUSRUN_TMPFOLDER} - if [ ! -f ${NM_TMPFOLDER}/.statusclean ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.statusclean)) -ge 3600 ]; then - for oldstatus_check in ${NM_TMPFOLDER}/status-*; do - [ -d "${oldstatus_check}" ] && [ $(expr $(date +%s) - $(stat -c %Y ${oldstatus_check})) -ge 1200 ] && rm -Rf ${oldstatus_check} - done - touch ${NM_TMPFOLDER}/.statusclean - fi + if [ ! -f ${NM_TMPFOLDER}/.statusclean ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.statusclean)) -ge 3600 ]; then + for oldstatus_check in ${NM_TMPFOLDER}/status-*; do + [ -d "${oldstatus_check}" ] && [ $(expr $(date +%s) - $(stat -c %Y ${oldstatus_check})) -ge 1200 ] && rm -Rf ${oldstatus_check} + done + touch ${NM_TMPFOLDER}/.statusclean + fi - if [ "${3}" == "email" ]; then - echo -e "\n\n" >> ${NM_LOGFOLDER}/status-check.lastrun - DOWNLIST report >> ${NM_LOGFOLDER}/status-check.lastrun - STATUSLOG="$(cat ${NM_LOGFOLDER}/status-check.lastrun)" - SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}" + if [ "${3}" == "email" ]; then + echo -e "\n\n" >> ${NM_LOGFOLDER}/status-check.lastrun + DOWNLIST report >> ${NM_LOGFOLDER}/status-check.lastrun + STATUSLOG="$(cat ${NM_LOGFOLDER}/status-check.lastrun)" + SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}" + fi + else + STATUS ${2} ${3} ${4} + rm -Rf ${STATUSRUN_TMPFOLDER} fi + else - STATUS ${2} ${3} - rm -Rf ${STATUSRUN_TMPFOLDER} + for oldstatus_check in ${NM_TMPFOLDER}/status-*; do + [ -d "${oldstatus_check}" ] && [ $(expr $(date +%s) - $(stat -c %Y ${oldstatus_check})) -ge 1200 ] && rm -Rf ${oldstatus_check} + done + touch ${NM_TMPFOLDER}/.statusclean + STATUSRUNS=($(find ${NM_TMPFOLDER}/* -maxdepth 1 -type d -iname "status-*" 2>/dev/null)) + if [ ${STATUSRUNS} -le 3 ]; then + ${NM_SCRIPT} status ${2} ${3} ${4} + exit 0 + else + if [ ! -f ${NM_TMPFOLDER}/.statusclean ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.overload)) -ge 600 ]; then + SENDNOTICE "NODEMGMT OVERLOAD" "Whooooaaaaa there cowboy, we got 3 threads running already.... easy killer!" 1 + touch ${NM_TMPFOLDER}/.overload + fi + exit 1 + fi fi fi;; scantimes) STATUS_SCANTIMES ${2} ${3} ${4} ${5};;