diff --git a/defaults.inc b/defaults.inc index ce21cbed..dfaa2684 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='5.3.10-11292024' +VERS='5.3.11-11292024' NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix ' diff --git a/inc/services.inc b/inc/services.inc index 16742c6b..53ee09e8 100755 --- a/inc/services.inc +++ b/inc/services.inc @@ -10,85 +10,44 @@ SINGLE_SERVER_SERVICE_CHECK(){ if [ ! -f ${NM_TMPFOLDER}/.skip ] && ([[ "$(declare -p ${1^^}_SINGLESRVR_SERVICES)" =~ "declare -a" ]] || [[ "$(declare -p ${1^^}_SINGLESRVR_DOCKERS)" =~ "declare -a" ]]); then if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) == *${NM_SINGLESRVR_IP[${1^^}]}* ]]; then if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^} ]; then - SENDNOTICE "Node-Balancing Notice" "Starting SingleServer Services on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on other Nodes" + SENDNOTICE "Node-Balancing Notice" "Starting SingleServer Services/Dockers on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on other Nodes" + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting SingleServer Services/Dockers on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on the other Nodes ..." >> ${NM_LOGFILE} touch ${NM_TMPFOLDER}/.singleserver.${1^^} - # fi - - - # gofor=false - # if [ "${1^^}" == "WEBxx" ] && [ ! -f ${NM_REPL_CHECK_LOC['web-data']}/mounted ]; then - # /bin/mount -a - # sleep 5s - # [ -f ${NM_REPL_CHECK_LOC["${1}-data"]}/mounted ] && gofor=true - # else - # gofor=true - # fi - # if [ "${gofor}" == "true" ]; then + for stc in "${!servicestocheck}"; do - if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting ${stc} on ${RUN_NODE_IP}, stopping on the other Nodes ..." >> ${NM_LOGFILE} - - [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket" - for nip in "${!hosts}"; do - if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then - ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} /bin/systemctl stop ${stc} - ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} rm -f ${NM_TMPFOLDER}/.singleserver.${1^^} - fi - done - sleep 3s - [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service" - [ "${stc}" = "headscale" ] && sleep 90s - /bin/systemctl start ${stc} & - fi + [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket" + for nip in "${!hosts}"; do + if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then + ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} /bin/systemctl stop ${stc} >/dev/null 2>&1 + ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} rm -f ${NM_TMPFOLDER}/.singleserver.${1^^} >/dev/null 2>&1 + fi + done + done + for docker in "${!dockerstocheck}"; do + for nip in "${!hosts}"; do + if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop ${docker} >/dev/null 2>&1 + [ "${docker}" == "authelia" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop auth_redis >/dev/null 2>&1 + fi + done done + sleep 20s + + for stc in "${!servicestocheck}"; do + [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket" + /bin/systemctl start ${stc} >/dev/null 2>&1 + done for docker in "${!dockerstocheck}"; do - if [ ! "$(docker ps -q -f name=${docker})" ]; then - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting ${docker} on ${RUN_NODE_IP}, stopping on the other Nodes ..." >> ${NM_LOGFILE} - - for nip in "${!hosts}"; do - if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then - #[ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker ps -q -f name=${docker})" ] && - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop ${docker} >/dev/null 2>&1 - [ "${docker}" == "authelia" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop auth_redis >/dev/null 2>&1 - fi - done - - sleep 3s - [ "${docker}" = "uptime-kuma" ] && sleep 15s - - [ "${docker}" == "authelia" ] && docker start auth_redis && sleep 2s >/dev/null 2>&1 - docker start ${docker} & >/dev/null 2>&1 - fi - - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE} - - # for nip in "${!hosts}"; do - # if [[ "${RUN_NODE_IP}" = *"${nip}"* ]]; then - # if [ "docker ps -a | grep ${docker})" ]; then - # [ "${docker}" == "authelia" ] && [ ! "$(docker ps -q -f name=auth_redis)" ] && docker start auth_redis && sleep 2s >/dev/null 2>&1 - # [ ! "$(docker ps -q -f name=${docker})" ] && docker start ${docker} & >/dev/null 2>&1 - # # elif [ "${NM_DOCKER_COMPOSE_LOC[${docker}]}" != "" ]; then - # # /usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - # # echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} - # fi - # - # - # else - # if [ "$(CHECK_HOST ${nip})" != "false" ]; then - # [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker ps -q -f name=${docker})" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop ${docker} >/dev/null 2>&1 - # [ "${docker}" == "authelia" ] && [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker ps -q -f name=auth_redis)" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} docker stop auth_redis >/dev/null 2>&1 - # fi - # fi - # done + [ "${docker}" == "authelia" ] && docker start auth_redis && sleep 2s >/dev/null 2>&1 + docker start ${docker} >/dev/null 2>&1 done echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE} fi - else if [ -f ${NM_TMPFOLDER}/.singleserver.${1^^} ]; then for stc in "${!servicestocheck}"; do @@ -96,7 +55,7 @@ SINGLE_SERVER_SERVICE_CHECK(){ /bin/systemctl stop ${stc} & >/dev/null 2>&1 done for docker in "${!dockerstocheck}"; do - [ "$(docker ps -q -f name=${docker})" ] && docker stop ${docker} & >/dev/null 2>&1 + docker stop ${docker} & >/dev/null 2>&1 done rm -f ${NM_TMPFOLDER}/.singleserver.${1^^} fi