voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-10-03 11:09:53 -05:00
parent 609e2fcb04
commit a9f8bf48db
2 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
defaults.inc
View File

@@ -9,7 +9,7 @@ NM_TMPFOLDER=${NM_FOLDER}/.tmp
NM_LOGFOLDER=${NM_FOLDER}/logs NM_LOGFOLDER=${NM_FOLDER}/logs
NM_LOGFILE=${NM_LOGFOLDER}/logfile NM_LOGFILE=${NM_LOGFOLDER}/logfile
RENOTIFY=3600 RENOTIFY_TIMEOUT=3600
[ ! -d ${NM_TMPFOLDER} ] && mkdir ${NM_TMPFOLDER} [ ! -d ${NM_TMPFOLDER} ] && mkdir ${NM_TMPFOLDER}
[ ! -d ${NM_LOGFOLDER} ] && mkdir ${NM_LOGFOLDER} [ ! -d ${NM_LOGFOLDER} ] && mkdir ${NM_LOGFOLDER}

14
inc/status.inc
View File

@@ -82,7 +82,7 @@ STATUS(){
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down) errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY_TIMEOUT} ]; then
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
fi fi
@@ -206,7 +206,7 @@ STATUS(){
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down) errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY_TIMEOUT} ]; then
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
@@ -279,7 +279,7 @@ STATUS(){
# #
# else # else
# errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down) # errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
# if [ $errtime -gt ${RENOTIFY} ]; then # if [ $errtime -gt ${RENOTIFY_TIMEOUT} ]; then
# if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then # if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime # mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
# fi # fi
@@ -378,7 +378,7 @@ STATUS(){
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down) errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY_TIMEOUT} ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
fi fi
@@ -420,13 +420,13 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi fi
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY_TIMEOUT} ]; then
touch ${NM_TMPFOLDER}/${nip}.down touch ${NM_TMPFOLDER}/${nip}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1 SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
fi fi
# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then # if [ $errtime -gt ${RENOTIFY_TIMEOUT} ] && [ "${EMAIL_NOTICE}" != "" ]; then
# echo "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_HOSTNAMES[${nip}]}[${nip}]" ${EMAIL_NOTICE} # echo "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_HOSTNAMES[${nip}]}[${nip}]" ${EMAIL_NOTICE}
# fi # fi
fi fi
@@ -518,7 +518,7 @@ STATUS(){
else else
snderrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)` snderrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)`
# echo "HERE: $snderrtime" # echo "HERE: $snderrtime"
if [ ${snderrtime} -gt ${RENOTIFY} ]; then if [ ${snderrtime} -gt ${RENOTIFY_TIMEOUT} ]; then
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)` toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)`
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]}) SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
It has been down for $(SHOW_TIME ${toterrtime})" 1 It has been down for $(SHOW_TIME ${toterrtime})" 1