voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-12-16 18:44:05 -06:00
parent 12bda7dcc6
commit 61806c636e
2 changed files with 56 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
defaults.inc
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env bash
VERS='5.1.23-12162023'
VERS='5.1.24-12162023'
NM_BETA=false
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck '

90
inc/status.inc
View File

@@ -630,46 +630,66 @@ STATUS_NODE(){
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ]; then
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
else
${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsCL[Green]}Running - Fixed"
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed"
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
else
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
else
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down))
if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
fi
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
fi
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down))
if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
fi
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
fi
fi
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) -gt 60 ]; then
[ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
else
${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
touch ${NM_TMPFOLDER}/${nip}~${docker}.fix
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsCL[Green]}Running - Fixed"
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed"
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
else
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1
touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
fi
fi
elif [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_SERVICE_DESC[${docker}]} - is down" 1
touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
fi
else
echo -en "${idsCL[Yellow]}Disabled - Skipping"
fi