voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2024-11-29 09:16:07 -06:00
parent 23415e69d7
commit cb324eff12
3 changed files with 55 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
defaults.inc
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
VERS='5.3.3-11202024'
VERS='5.3.4-11292024'
NM_BETA=false
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix '

73
inc/status.inc
View File

@@ -582,10 +582,17 @@ STATUS_NODE(){
else
SSCHK=primary
fi
elif ([ "${nip}" == "10.2.1.51" ] || [ "${nip}" == "10.2.1.240" ]) && [ "${srvc}" == "keepalived" ]; then
SSCHK=false
elif ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
SSCHK=false
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
else
SSCHK=true
fi
if [ ${SSCHK} != false ]; then
@@ -684,8 +691,14 @@ STATUS_NODE(){
else
SDCHK=primary
fi
elif ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
SDCHK=false
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
else
SDCHK=true
fi
if [ ${SDCHK} != false ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
@@ -698,36 +711,36 @@ STATUS_NODE(){
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, will try and fix in 1min" >> ${NM_LOGFILE}
echo -en "${idsCL[Red]}Not Running - will try and fix in 1min!"
# elif ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) -gt 50 ]) || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.fix)) -ge 3600 ]); then
# [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
# if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
# ${NCMD} docker start ${docker} >/dev/null 2>&1
# echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
# else
# ${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
# echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
# fi
# touch ${NM_TMPFOLDER}/${nip}~${docker}.fix
# sleep 10s
# if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
# [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Green]}Running - Fixed"
# [ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed"
# rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
# echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
#
# else
# if [ "${STATUS_ACTION}" != "check" ]; then
# [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1
# touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
# touch ${NM_TMPFOLDER}/${nip}~${docker}.down
# echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
#
# fi
# fi
# elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then
elif ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) -gt 50 ]) || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.fix)) -ge 3600 ]); then
[ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
else
${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
touch ${NM_TMPFOLDER}/${nip}~${docker}.fix
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Green]}Running - Fixed"
[ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed"
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
else
if [ "${STATUS_ACTION}" != "check" ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1
touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
fi
fi
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then
# elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
touch ${NM_TMPFOLDER}/${nip}~${docker}.down

11
nodemgmt-scripts.sh
View File

@@ -1162,6 +1162,15 @@ SETUPSSH(){
fi
done
}
SKIP_SERVER_CHECKS(){
if [ ! -f ${NM_TMPFOLDER}/.skip ]; then
touch ${NM_TMPFOLDER}/.skip
echo -e "${idsCL[Yellow]}Server will be skipped from Node-Manager Status Checks ...${idsCL[Default]}"
else
rm -f ${NM_TMPFOLDER}/.skip
echo -e "${idsCL[Green]}Server has been re-enabled for Node-Manager Status Checks ...${idsCL[Default]}"
fi
}
SETUPSSH_BACK(){
echo -en "\n${idsCL[LightCyan]}Checking for Public SSH Key back to Node-Manager ... ${idsCL[Default]}"
if [ "$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@10.10.10.60 echo ok 2>&1)" == "ok" ]; then
@@ -1390,6 +1399,8 @@ GUI(){
tmp=tmp
;;
skip) SKIP_SERVER_CHECKS;;
nginxupdate)
echo
for nfile in ${NM_NGINXPATH}/sites-enabled/*.conf; do