update
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
VERS='5.5.18-06102025'
|
VERS='5.5.19-06112025'
|
||||||
NM_BETA=false
|
NM_BETA=false
|
||||||
|
|
||||||
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix '
|
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix '
|
||||||
|
|||||||
158
inc/services.inc
158
inc/services.inc
@@ -1,83 +1,113 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
SINGLE_SERVER_SERVICE_CHECK(){
|
SINGLE_SERVER_SERVICE_CHECK(){
|
||||||
servicestocheck=${1^^}_SINGLESRVR_SERVICES[@]
|
[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.check)) -ge 1200 ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check
|
||||||
dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@]
|
|
||||||
hosts=${1^^}_HOSTS[@]
|
|
||||||
|
|
||||||
[ "${2^^}" == "START" ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
|
if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check ]; then
|
||||||
|
touch ${NM_TMPFOLDER}/.singleserver.${1^^}.check
|
||||||
|
|
||||||
# rm -f ${NM_TMPFOLDER}/.singleserver*
|
servicestocheck=${1^^}_SINGLESRVR_SERVICES[@]
|
||||||
|
dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@]
|
||||||
|
hosts=${1^^}_HOSTS[@]
|
||||||
|
|
||||||
|
[ "${2^^}" == "START" ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# rm -f ${NM_TMPFOLDER}/.singleserver*
|
||||||
|
|
||||||
if [ ! -f ${NM_TMPFOLDER}/.skip ] && ([[ "$(declare -p ${1^^}_SINGLESRVR_SERVICES)" =~ "declare -a" ]] || [[ "$(declare -p ${1^^}_SINGLESRVR_DOCKERS)" =~ "declare -a" ]]); then
|
if [ ! -f ${NM_TMPFOLDER}/.skip ] && ([[ "$(declare -p ${1^^}_SINGLESRVR_SERVICES)" =~ "declare -a" ]] || [[ "$(declare -p ${1^^}_SINGLESRVR_DOCKERS)" =~ "declare -a" ]]); then
|
||||||
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) == *${NM_SINGLESRVR_IP[${1^^}]}* ]]; then
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) == *${NM_SINGLESRVR_IP[${1^^}]}* ]]; then
|
||||||
# if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.primary)) -ge 300 ]; then
|
# if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.primary)) -ge 300 ]; then
|
||||||
# if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
|
# if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
|
||||||
if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
|
if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
|
||||||
touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
|
touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
|
||||||
SENDNOTICE "'${NM_NODETYPES[${1^^}]}' Node Switch" "Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on other Nodes"
|
SENDNOTICE "'${NM_NODETYPES[${1^^}]}' Node Switch" "Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on other Nodes"
|
||||||
fi
|
fi
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on the other Nodes ..." >> ${NM_LOGFILE}
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on the other Nodes ..." >> ${NM_LOGFILE}
|
||||||
for stc in "${!servicestocheck}"; do
|
for stc in "${!servicestocheck}"; do
|
||||||
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
|
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
|
||||||
for nip in "${!hosts}"; do
|
for nip in "${!hosts}"; do
|
||||||
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
|
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
|
||||||
${SSHCMD} root@${nip} 'systemctl is-active --quiet '${stc}' && /bin/systemctl stop '${stc} >/dev/null 2>&1
|
${SSHCMD} root@${nip} 'systemctl is-active --quiet '${stc}' && /bin/systemctl stop '${stc} >/dev/null 2>&1
|
||||||
${SSHCMD} root@${nip} '[ -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary ] && rm -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary' >/dev/null 2>&1
|
${SSHCMD} root@${nip} '[ -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary ] && rm -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary' >/dev/null 2>&1
|
||||||
fi
|
fi
|
||||||
|
done
|
||||||
done
|
done
|
||||||
|
for docker in "${!dockerstocheck}"; do
|
||||||
|
for nip in "${!hosts}"; do
|
||||||
|
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
|
||||||
|
${SSHCMD} root@${nip} '[ "$(docker ps -q -f name='${docker}')" ] && docker stop '${docker} >/dev/null 2>&1
|
||||||
|
[ "${docker}" == "authelia" ] && ${SSHCMD} root@${nip} '[ "$(docker ps -q -f name=auth_redis)" ] && docker stop auth_redis' >/dev/null 2>&1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done
|
||||||
|
sleep 20s
|
||||||
|
# else
|
||||||
|
touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
|
||||||
|
# fi
|
||||||
|
|
||||||
|
for stc in "${!servicestocheck}"; do
|
||||||
|
if ! systemctl is-active --quiet ${stc}; then
|
||||||
|
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
|
||||||
|
/bin/systemctl start ${stc} >/dev/null 2>&1
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
for docker in "${!dockerstocheck}"; do
|
for docker in "${!dockerstocheck}"; do
|
||||||
for nip in "${!hosts}"; do
|
if [ ! "$(docker ps -q -f name=${docker})" ]; then
|
||||||
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
|
[ "${docker}" == "authelia" ] && docker start auth_redis && sleep 2s >/dev/null 2>&1
|
||||||
${SSHCMD} root@${nip} '[ "$(docker ps -q -f name='${docker}')" ] && docker stop '${docker} >/dev/null 2>&1
|
docker start ${docker} >/dev/null 2>&1
|
||||||
[ "${docker}" == "authelia" ] && ${SSHCMD} root@${nip} '[ "$(docker ps -q -f name=auth_redis)" ] && docker stop auth_redis' >/dev/null 2>&1
|
fi
|
||||||
fi
|
|
||||||
done
|
|
||||||
done
|
done
|
||||||
sleep 20s
|
[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
|
||||||
# else
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE}
|
||||||
touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
|
|
||||||
# fi
|
# fi
|
||||||
|
|
||||||
for stc in "${!servicestocheck}"; do
|
if [ "${1^^}" == "LPD" ]; then
|
||||||
if ! systemctl is-active --quiet ${stc}; then
|
/usr/local/bin/nmg skip on >/dev/null 2>&1
|
||||||
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
|
completed=false; scanstart=$(date +%s)
|
||||||
/bin/systemctl start ${stc} >/dev/null 2>&1
|
until [ "${completed}" == "true" ]; do
|
||||||
fi
|
dnstest=$(dig +short www.google.com @127.0.0.1)
|
||||||
done
|
if [ "${dnstest}" != "" ]; then
|
||||||
for docker in "${!dockerstocheck}"; do
|
completed=true
|
||||||
if [ ! "$(docker ps -q -f name=${docker})" ]; then
|
break
|
||||||
[ "${docker}" == "authelia" ] && docker start auth_redis && sleep 2s >/dev/null 2>&1
|
|
||||||
docker start ${docker} >/dev/null 2>&1
|
elif [ $(($(date +%s)-scanstart)) -gt 1200 ]; then
|
||||||
fi
|
SENDNOTICE "${NM_HOSTNAMES[${RUN_NODE_IP}]} DNS Startup Failure" "${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP})
|
||||||
done
|
Server did not start serving DNS requests within 20mins you moron...." 1
|
||||||
[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
|
killall bash
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE}
|
exit 1
|
||||||
# fi
|
fi
|
||||||
|
sleep 15s
|
||||||
|
done
|
||||||
|
/usr/local/bin/nmg skip off >/dev/null 2>&1
|
||||||
|
fi
|
||||||
|
|
||||||
else
|
else
|
||||||
# if [ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary)) -ge 300 ]; then
|
# if [ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary)) -ge 300 ]; then
|
||||||
[ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && touch ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
|
[ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && touch ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
|
||||||
for stc in "${!servicestocheck}"; do
|
for stc in "${!servicestocheck}"; do
|
||||||
if [ systemctl is-active --quiet ${stc} ]; then
|
if [ systemctl is-active --quiet ${stc} ]; then
|
||||||
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
|
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
|
||||||
/bin/systemctl stop ${stc} & >/dev/null 2>&1
|
/bin/systemctl stop ${stc} & >/dev/null 2>&1
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
for docker in "${!dockerstocheck}"; do
|
for docker in "${!dockerstocheck}"; do
|
||||||
[ "$(docker ps -q -f name=${docker})" ] && docker stop ${docker} & >/dev/null 2>&1
|
[ "$(docker ps -q -f name=${docker})" ] && docker stop ${docker} & >/dev/null 2>&1
|
||||||
done
|
done
|
||||||
[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
|
[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
|
||||||
# fi
|
# fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
[ "${1^^}" == "WEB" ] && [ "$(systemctl is-active nginx)" != "active" ] && ${NM_SCRIPT} service web restart ${RUN_NODE_IP}
|
[ "${1^^}" == "WEB" ] && [ "$(systemctl is-active nginx)" != "active" ] && ${NM_SCRIPT} service web restart ${RUN_NODE_IP}
|
||||||
|
|
||||||
elif [ "${2^^}" == "START" ] && [ -f ${NM_TMPFOLDER}/.skip ]; then
|
elif [ "${2^^}" == "START" ] && [ -f ${NM_TMPFOLDER}/.skip ]; then
|
||||||
systemctl stop keepalived >/dev/null 2>&1
|
systemctl stop keepalived >/dev/null 2>&1
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check
|
||||||
fi
|
fi
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
SERVICE_MGMT(){
|
SERVICE_MGMT(){
|
||||||
|
|||||||
Reference in New Issue
Block a user