diff --git a/defaults.inc b/defaults.inc index 0b4973b1..0494781d 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='5.3.3-11202024' +VERS='5.3.4-11292024' NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix ' diff --git a/inc/status.inc b/inc/status.inc index 299c311f..d24c41f5 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -582,10 +582,17 @@ STATUS_NODE(){ else SSCHK=primary fi + elif ([ "${nip}" == "10.2.1.51" ] || [ "${nip}" == "10.2.1.240" ]) && [ "${srvc}" == "keepalived" ]; then SSCHK=false + + elif ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then + SSCHK=false + rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* + else SSCHK=true + fi if [ ${SSCHK} != false ]; then @@ -684,8 +691,14 @@ STATUS_NODE(){ else SDCHK=primary fi + + elif ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then + SDCHK=false + rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* + else SDCHK=true + fi if [ ${SDCHK} != false ]; then [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: " @@ -698,36 +711,36 @@ STATUS_NODE(){ echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, will try and fix in 1min" >> ${NM_LOGFILE} echo -en "${idsCL[Red]}Not Running - will try and fix in 1min!" - # elif ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) -gt 50 ]) || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.fix)) -ge 3600 ]); then - # [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime - # if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then - # ${NCMD} docker start ${docker} >/dev/null 2>&1 - # echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} - # else - # ${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - # echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} - # fi - # touch ${NM_TMPFOLDER}/${nip}~${docker}.fix - # sleep 10s - # if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - # [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Green]}Running - Fixed" - # [ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed" - # rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* - # echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} - # - # else - # if [ "${STATUS_ACTION}" != "check" ]; then - # [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running - Could Not Fix!" - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1 - # touch ${NM_TMPFOLDER}/${nip}~${docker}.sent - # touch ${NM_TMPFOLDER}/${nip}~${docker}.down - # echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} - # - # fi - # fi - - # elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then - elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then + elif ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) -gt 50 ]) || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.fix)) -ge 3600 ]); then + [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime + if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then + ${NCMD} docker start ${docker} >/dev/null 2>&1 + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} + else + ${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} + fi + touch ${NM_TMPFOLDER}/${nip}~${docker}.fix + sleep 10s + if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Green]}Running - Fixed" + [ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed" + rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} + + else + if [ "${STATUS_ACTION}" != "check" ]; then + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running - Could Not Fix!" + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1 + touch ${NM_TMPFOLDER}/${nip}~${docker}.sent + touch ${NM_TMPFOLDER}/${nip}~${docker}.down + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} + + fi + fi + + elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then + # elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)) [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" touch ${NM_TMPFOLDER}/${nip}~${docker}.down diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 90bf52db..6ee638c8 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -1162,6 +1162,15 @@ SETUPSSH(){ fi done } +SKIP_SERVER_CHECKS(){ + if [ ! -f ${NM_TMPFOLDER}/.skip ]; then + touch ${NM_TMPFOLDER}/.skip + echo -e "${idsCL[Yellow]}Server will be skipped from Node-Manager Status Checks ...${idsCL[Default]}" + else + rm -f ${NM_TMPFOLDER}/.skip + echo -e "${idsCL[Green]}Server has been re-enabled for Node-Manager Status Checks ...${idsCL[Default]}" + fi +} SETUPSSH_BACK(){ echo -en "\n${idsCL[LightCyan]}Checking for Public SSH Key back to Node-Manager ... ${idsCL[Default]}" if [ "$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@10.10.10.60 echo ok 2>&1)" == "ok" ]; then @@ -1390,6 +1399,8 @@ GUI(){ tmp=tmp ;; + skip) SKIP_SERVER_CHECKS;; + nginxupdate) echo for nfile in ${NM_NGINXPATH}/sites-enabled/*.conf; do