voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-12-05 18:16:13 -06:00
parent 673b83edf5
commit 56f5579d88
4 changed files with 23 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
defaults.inc
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env bash
VERS='4.17.53-12052023'
VERS='4.17.54-12052023'
NM_BETA=false
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck '

8
inc/services.inc
View File

@@ -483,9 +483,9 @@ CHECK_DOCKER_SERVICES(){
fi
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
else
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
fi
rm -f ${NM_TMPFOLDER}/${nip}.down
rm -f ${NM_TMPFOLDER}/${nip}.errtime
@@ -527,10 +527,10 @@ CHECK_DOCKER_SERVICES(){
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
else
errtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)`
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
fi
fi

32
inc/status.inc
View File

@@ -97,12 +97,12 @@ STATUS(){
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else
errtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)`
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down))
if [ $errtime -gt ${NM_RENOTIFY_TIMEOUT} ]; then
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
fi
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
@@ -115,9 +115,9 @@ STATUS(){
fi
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
else
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down))
fi
rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
@@ -204,7 +204,7 @@ STATUS(){
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.sent ]; then
[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)` || toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)`
[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
rm -f ${NM_TMPFOLDER}/${nip}.*
@@ -270,10 +270,10 @@ STATUS(){
[ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} &
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then
errtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)`
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
if [ $errtime -gt ${NM_RENOTIFY_TIMEOUT} ]; then
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime))
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
@@ -287,7 +287,7 @@ STATUS(){
else
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)` || toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)`
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
@@ -371,12 +371,12 @@ STATUS(){
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
else
errtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)`
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down))
if [ $errtime -gt ${NM_RENOTIFY_TIMEOUT} ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
fi
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
@@ -408,7 +408,7 @@ STATUS(){
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
if [ "${checkhost}" != "false" ]; then
echo -e "${idsCL[Red]} - Node is online, but SSL is down!${idsCL[Default]}"
@@ -419,8 +419,8 @@ STATUS(){
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
[ "${checkhostssl}" != "ok" ] && echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
else
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)`
errtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
if [ "${ST_ACTION}" != "check" ]; then
if [ "${checkhost}" != "false" ]; then
echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
@@ -530,10 +530,10 @@ STATUS(){
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
else
snderrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)`
snderrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down))
# echo "HERE: $snderrtime"
if [ ${snderrtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime))
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
It has been down for $(SHOW_TIME ${toterrtime})" 1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, has now been down for: $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
@@ -545,7 +545,7 @@ It has been down for $(SHOW_TIME ${toterrtime})" 1
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}"
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ]; then
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime))
SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}

4
nodemgmt-scripts.sh
View File

@@ -684,9 +684,9 @@ DOWNLIST(){
fi
if [ -f ${NM_TMPFOLDER}/${ditem}.errtime ]; then
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ditem}.errtime)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ditem}.errtime))
else
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ditem}.down)`
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ditem}.down))
fi
c=0; spc=''; spc1=${cw}-${#itemname}