diff --git a/defaults.inc b/defaults.inc index dfaa2684..564552d3 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='5.3.11-11292024' +VERS='5.3.12-11292024' NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix ' diff --git a/inc/services.inc b/inc/services.inc index 53ee09e8..069fdc68 100755 --- a/inc/services.inc +++ b/inc/services.inc @@ -5,51 +5,51 @@ SINGLE_SERVER_SERVICE_CHECK(){ dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@] hosts=${1^^}_HOSTS[@] - [ "${2^^}" == "START" ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^} + [ "${2^^}" == "START" ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary if [ ! -f ${NM_TMPFOLDER}/.skip ] && ([[ "$(declare -p ${1^^}_SINGLESRVR_SERVICES)" =~ "declare -a" ]] || [[ "$(declare -p ${1^^}_SINGLESRVR_DOCKERS)" =~ "declare -a" ]]); then if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) == *${NM_SINGLESRVR_IP[${1^^}]}* ]]; then - if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^} ]; then - SENDNOTICE "Node-Balancing Notice" "Starting SingleServer Services/Dockers on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on other Nodes" - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting SingleServer Services/Dockers on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on the other Nodes ..." >> ${NM_LOGFILE} - touch ${NM_TMPFOLDER}/.singleserver.${1^^} - - for stc in "${!servicestocheck}"; do - [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket" - for nip in "${!hosts}"; do - if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then - ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} /bin/systemctl stop ${stc} >/dev/null 2>&1 - ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} rm -f ${NM_TMPFOLDER}/.singleserver.${1^^} >/dev/null 2>&1 - fi + if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.primary)) -ge 900 ]; then + if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then + touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary + SENDNOTICE "Node-Balancing Notice" "Starting SingleServer Services/Dockers on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on other Nodes" + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting SingleServer Services/Dockers on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on the other Nodes ..." >> ${NM_LOGFILE} + for stc in "${!servicestocheck}"; do + [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket" + for nip in "${!hosts}"; do + if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then + ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} /bin/systemctl stop ${stc} >/dev/null 2>&1 + ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary >/dev/null 2>&1 + fi + done done - done - for docker in "${!dockerstocheck}"; do - for nip in "${!hosts}"; do - if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop ${docker} >/dev/null 2>&1 - [ "${docker}" == "authelia" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop auth_redis >/dev/null 2>&1 - fi + for docker in "${!dockerstocheck}"; do + for nip in "${!hosts}"; do + if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop ${docker} >/dev/null 2>&1 + [ "${docker}" == "authelia" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop auth_redis >/dev/null 2>&1 + fi + done done - done - - sleep 20s + sleep 20s + else + touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary + fi for stc in "${!servicestocheck}"; do [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket" /bin/systemctl start ${stc} >/dev/null 2>&1 done - for docker in "${!dockerstocheck}"; do [ "${docker}" == "authelia" ] && docker start auth_redis && sleep 2s >/dev/null 2>&1 docker start ${docker} >/dev/null 2>&1 done echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE} - fi else - if [ -f ${NM_TMPFOLDER}/.singleserver.${1^^} ]; then + if [ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary)) -ge 900 ]; then for stc in "${!servicestocheck}"; do [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service" /bin/systemctl stop ${stc} & >/dev/null 2>&1 @@ -57,7 +57,8 @@ SINGLE_SERVER_SERVICE_CHECK(){ for docker in "${!dockerstocheck}"; do docker stop ${docker} & >/dev/null 2>&1 done - rm -f ${NM_TMPFOLDER}/.singleserver.${1^^} + rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary + touch ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary fi fi