From c7fd86be6b5a1c6a8c58a3deadea2e2e7382597d Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Wed, 11 Jun 2025 23:14:37 -0500
Subject: [PATCH] update

---
 defaults.inc     |   2 +-
 inc/services.inc | 158 ++++++++++++++++++++++++++++-------------------
 2 files changed, 95 insertions(+), 65 deletions(-)

diff --git a/defaults.inc b/defaults.inc
index dc61b84b..ce5ca1a4 100755
--- a/defaults.inc
+++ b/defaults.inc
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-VERS='5.5.18-06102025'
+VERS='5.5.19-06112025'
 NM_BETA=false
 
 noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix '
diff --git a/inc/services.inc b/inc/services.inc
index dba6fcbe..7ef2efcf 100755
--- a/inc/services.inc
+++ b/inc/services.inc
@@ -1,83 +1,113 @@
 #!/usr/bin/env bash
 
 SINGLE_SERVER_SERVICE_CHECK(){
-	servicestocheck=${1^^}_SINGLESRVR_SERVICES[@]
-	dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@]
-	hosts=${1^^}_HOSTS[@]
+	[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.check)) -ge 1200 ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check
 	
-	[ "${2^^}" == "START" ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
+	if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check ]; then
+		touch ${NM_TMPFOLDER}/.singleserver.${1^^}.check
 	
-	# rm -f ${NM_TMPFOLDER}/.singleserver*
+		servicestocheck=${1^^}_SINGLESRVR_SERVICES[@]
+		dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@]
+		hosts=${1^^}_HOSTS[@]
+	
+		[ "${2^^}" == "START" ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
+
+	
+	
+		# rm -f ${NM_TMPFOLDER}/.singleserver*
 			
-	if [ ! -f ${NM_TMPFOLDER}/.skip ] && ([[ "$(declare -p ${1^^}_SINGLESRVR_SERVICES)" =~ "declare -a" ]] || [[ "$(declare -p ${1^^}_SINGLESRVR_DOCKERS)" =~ "declare -a" ]]); then	
-		if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) == *${NM_SINGLESRVR_IP[${1^^}]}* ]]; then
-			# if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.primary)) -ge 300 ]; then
-			# 	if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
-					if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
-						touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
-						SENDNOTICE "'${NM_NODETYPES[${1^^}]}' Node Switch" "Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on other Nodes"
-					fi
-					echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on the other Nodes ..." >> ${NM_LOGFILE}
-					for stc in "${!servicestocheck}"; do
-						[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
-						for nip in "${!hosts}"; do
-							if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
-								${SSHCMD} root@${nip} 'systemctl is-active --quiet '${stc}' && /bin/systemctl stop '${stc} >/dev/null 2>&1
-								${SSHCMD} root@${nip} '[ -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary ] && rm -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary' >/dev/null 2>&1
-							fi
+		if [ ! -f ${NM_TMPFOLDER}/.skip ] && ([[ "$(declare -p ${1^^}_SINGLESRVR_SERVICES)" =~ "declare -a" ]] || [[ "$(declare -p ${1^^}_SINGLESRVR_DOCKERS)" =~ "declare -a" ]]); then	
+			if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) == *${NM_SINGLESRVR_IP[${1^^}]}* ]]; then
+				# if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.primary)) -ge 300 ]; then
+				# 	if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
+						if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
+							touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
+							SENDNOTICE "'${NM_NODETYPES[${1^^}]}' Node Switch" "Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on other Nodes"
+						fi
+						echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on the other Nodes ..." >> ${NM_LOGFILE}
+						for stc in "${!servicestocheck}"; do
+							[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
+							for nip in "${!hosts}"; do
+								if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
+									${SSHCMD} root@${nip} 'systemctl is-active --quiet '${stc}' && /bin/systemctl stop '${stc} >/dev/null 2>&1
+									${SSHCMD} root@${nip} '[ -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary ] && rm -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary' >/dev/null 2>&1
+								fi
+							done
 						done
+						for docker in "${!dockerstocheck}"; do
+							for nip in "${!hosts}"; do
+								if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
+									${SSHCMD} root@${nip} '[ "$(docker ps -q -f name='${docker}')" ] && docker stop '${docker} >/dev/null 2>&1
+									[ "${docker}" == "authelia" ] && ${SSHCMD} root@${nip} '[ "$(docker ps -q -f name=auth_redis)" ] && docker stop auth_redis' >/dev/null 2>&1
+								fi
+							done
+						done
+						sleep 20s
+					# else
+						touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
+					# fi
+				
+					for stc in "${!servicestocheck}"; do
+						if ! systemctl is-active --quiet ${stc}; then
+							[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
+							/bin/systemctl start ${stc} >/dev/null 2>&1
+						fi
 					done
 					for docker in "${!dockerstocheck}"; do
-						for nip in "${!hosts}"; do
-							if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
-								${SSHCMD} root@${nip} '[ "$(docker ps -q -f name='${docker}')" ] && docker stop '${docker} >/dev/null 2>&1
-								[ "${docker}" == "authelia" ] && ${SSHCMD} root@${nip} '[ "$(docker ps -q -f name=auth_redis)" ] && docker stop auth_redis' >/dev/null 2>&1
-							fi
-						done
+						if [ ! "$(docker ps -q -f name=${docker})" ]; then
+							[ "${docker}" == "authelia" ] && docker start auth_redis && sleep 2s >/dev/null 2>&1
+							docker start ${docker} >/dev/null 2>&1
+						fi
 					done
-					sleep 20s
-				# else
-					touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
+					[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
+					echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE}
 				# fi
 				
-				for stc in "${!servicestocheck}"; do
-					if ! systemctl is-active --quiet ${stc}; then
-						[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
-						/bin/systemctl start ${stc} >/dev/null 2>&1
-					fi
-				done
-				for docker in "${!dockerstocheck}"; do
-					if [ ! "$(docker ps -q -f name=${docker})" ]; then
-						[ "${docker}" == "authelia" ] && docker start auth_redis && sleep 2s >/dev/null 2>&1
-						docker start ${docker} >/dev/null 2>&1
-					fi
-				done
-				[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
-				echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE}
-			# fi
+				if [ "${1^^}" == "LPD" ]; then
+					/usr/local/bin/nmg skip on >/dev/null 2>&1
+					completed=false; scanstart=$(date +%s)
+					until [ "${completed}" == "true" ]; do
+						dnstest=$(dig +short www.google.com @127.0.0.1)
+						if [ "${dnstest}" != "" ]; then
+							completed=true
+							break
+							
+						elif [ $(($(date +%s)-scanstart)) -gt 1200 ]; then
+							SENDNOTICE "${NM_HOSTNAMES[${RUN_NODE_IP}]} DNS Startup Failure" "${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP})
+Server did not start serving DNS requests within 20mins you moron...." 1
+							killall bash
+							exit 1
+						fi
+						sleep 15s
+					done
+					/usr/local/bin/nmg skip off >/dev/null 2>&1
+				fi
 
-		else
-			# if [ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary)) -ge 300 ]; then
-				[ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && touch ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
-				for stc in "${!servicestocheck}"; do
-					if [ systemctl is-active --quiet ${stc} ]; then
-						[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
-						/bin/systemctl stop ${stc} & >/dev/null 2>&1
-					fi
-				done
-				for docker in "${!dockerstocheck}"; do
-					[ "$(docker ps -q -f name=${docker})" ] && docker stop ${docker} & >/dev/null 2>&1
-				done
-				[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
-			# fi
-		fi
+			else
+				# if [ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary)) -ge 300 ]; then
+					[ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && touch ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
+					for stc in "${!servicestocheck}"; do
+						if [ systemctl is-active --quiet ${stc} ]; then
+							[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
+							/bin/systemctl stop ${stc} & >/dev/null 2>&1
+						fi
+					done
+					for docker in "${!dockerstocheck}"; do
+						[ "$(docker ps -q -f name=${docker})" ] && docker stop ${docker} & >/dev/null 2>&1
+					done
+					[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
+				# fi
+			fi
 	
-		[ "${1^^}" == "WEB" ] && [ "$(systemctl is-active nginx)" != "active" ] && ${NM_SCRIPT} service web restart ${RUN_NODE_IP}
+			[ "${1^^}" == "WEB" ] && [ "$(systemctl is-active nginx)" != "active" ] && ${NM_SCRIPT} service web restart ${RUN_NODE_IP}
 		
-	elif [ "${2^^}" == "START" ] && [ -f ${NM_TMPFOLDER}/.skip ]; then
-		systemctl stop keepalived >/dev/null 2>&1
+		elif [ "${2^^}" == "START" ] && [ -f ${NM_TMPFOLDER}/.skip ]; then
+			systemctl stop keepalived >/dev/null 2>&1
+		fi
+		
+		
+		rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check
 	fi
-	
 }
 
 SERVICE_MGMT(){