diff --git a/defaults.inc b/defaults.inc index 74b51fd6..f4eb9b0e 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ #!/usr/bin/env bash -VERS='5.1.23-12162023' +VERS='5.1.24-12162023' NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck ' diff --git a/inc/status.inc b/inc/status.inc index 197359cf..f9369000 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -630,46 +630,66 @@ STATUS_NODE(){ if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ]; then - if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then - ${NCMD} docker start ${docker} >/dev/null 2>&1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} - else - ${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} - fi - sleep 10s - if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "${STATUS_ACTION}" != "check" ]; then - echo -en "${idsCL[Green]}Running - Fixed" - fi - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed" - rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} + + + if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then else - if [ "${STATUS_ACTION}" != "check" ]; then - echo -en "${idsCL[Red]}Not Running - Could Not Fix!" - - if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1 - touch ${NM_TMPFOLDER}/${nip}~${docker}.down - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} - else - errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) - if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then - if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then - mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime - fi - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)) - if [ "${STATUS_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - touch ${NM_TMPFOLDER}/${nip}~${docker}.down - fi + errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) + if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then + if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then + mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime fi - + toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)) + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + touch ${NM_TMPFOLDER}/${nip}~${docker}.down fi fi + + + if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1 + touch ${NM_TMPFOLDER}/${nip}~${docker}.down + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} + + elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) -gt 60 ]; then + [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime + if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then + ${NCMD} docker start ${docker} >/dev/null 2>&1 + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} + else + ${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} + fi + touch ${NM_TMPFOLDER}/${nip}~${docker}.fix + sleep 10s + if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then + echo -en "${idsCL[Green]}Running - Fixed" + fi + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed" + rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} + + else + if [ "${STATUS_ACTION}" != "check" ]; then + echo -en "${idsCL[Red]}Not Running - Could Not Fix!" + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1 + touch ${NM_TMPFOLDER}/${nip}~${docker}.sent + touch ${NM_TMPFOLDER}/${nip}~${docker}.down + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} + + fi + fi + + elif [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then + toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)) + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + touch ${NM_TMPFOLDER}/${nip}~${docker}.down + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_SERVICE_DESC[${docker}]} - is down" 1 + touch ${NM_TMPFOLDER}/${nip}~${docker}.sent + fi + else echo -en "${idsCL[Yellow]}Disabled - Skipping" fi