From 519a01f3a3fe57d611f55ccedf9df02ec00dd138 Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Mon, 1 May 2023 20:47:52 -0500
Subject: [PATCH] update

---
 defaults.inc   |  4 +++
 inc/status.inc | 71 +++++++++++++++++++++++++++++++++++---------------
 run.sh         |  9 ++++---
 3 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/defaults.inc b/defaults.inc
index a51b3062..e5adae0a 100755
--- a/defaults.inc
+++ b/defaults.inc
@@ -32,10 +32,14 @@ WEBDOCKER=phpmyadmin,phppgadmin
 noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck '
 CERT_DAEMON='/snap/bin/certbot'
 FOLDER=/opt/idssys/nodemgmt
+TMPFOLDER=${FOLDER}/.tmp
 SCRIPT=${FOLDER}/nodemgmt-scripts.sh
 LOGFILE=${FOLDER}/logfile
 RENOTIFY=900
 
+[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER}
+
+
 NM_SRVCOPT=(start stop restart reload enable disable)
 
 #MYSQL_REPL_CHECK=( )
diff --git a/inc/status.inc b/inc/status.inc
index fcc0737e..de2006f3 100755
--- a/inc/status.inc
+++ b/inc/status.inc
@@ -413,6 +413,46 @@ STATUS(){
 			echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
 			DIVIDER; echo -e "${idsST[Reset]}"
 		fi
+		
+		
+		for NTYPE in "${NODE_TYPES[@]}"; do
+			PH=${NTYPE}_HOSTS[0]
+			PH_CMD="ssh root@${!PH}"
+			var=${NTYPE}_REPL_CHECK[@]
+			if [ ! -z ${!var+x} ]; then
+				nid=1
+				var=${NTYPE}_HOSTS[@]
+				for nip in "${!var}"; do
+					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
+						if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
+							var2=${NTYPE}_REPL_CHECK[@]
+							for rcheck in "${!var2}"; do
+								if [ "${1}" != "check" ]; then
+									timeout=`date --date='1 minutes' +%s`
+								else
+									timeout=`date --date='2 minutes' +%s`
+								fi
+								checked=false
+								until [ "${checked}" == "" ]; do
+									if [ "${PH_CMD}" == "" ]; then
+										ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
+									else
+										ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
+									fi
+									[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
+								done
+								[ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good
+							done
+
+						fi
+					fi
+					nid=`expr $nid + 1`
+				done
+			fi
+		done
+		
+		
+				
 		# replcheckstart=`date +%s`
 		# echo "$(date) - STARTING REPL CHECK -	$(SHOW_TIME $(expr `date +%s` - $replstart) s)" >> ${FOLDER}/log.repl
 		for NTYPE in "${NODE_TYPES[@]}"; do
@@ -657,6 +697,14 @@ STATUS(){
 
 
 REPLCHECK(){
+	nip=
+	rcheck=
+	PH_CMD=
+	if [ "${1}" != "check" ]; then
+		timeout=`date --date='1 minutes' +%s`
+	else
+		timeout=`date --date='2 minutes' +%s`
+	fi
 	checked=false
 	until [ "${checked}" == "" ]; do
 		if [ "${PH_CMD}" == "" ]; then
@@ -664,26 +712,7 @@ REPLCHECK(){
 		else
 			ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
 		fi
-		if [ "`date +%s`" -gt "${timeout}" ]; then
-			timeout=true
-			break
-		fi
+		[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
 	done
-	if [ "${timeout}" == "true" ]; then
-		[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]} - ${rrmsg}"
-		if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-			touch ${FOLDER}/${nip}-${rcheck}.down
-			SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
-			echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
-		fi
-		# echo "$(date) - TIMEOUT - ${rcheck} -	${rrmsg}" >> ${FOLDER}/log.repl
-	else
-		[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]} - ${rrmsg}"
-		if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
-			rm -f ${FOLDER}/${nip}-${rcheck}.down
-			SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
-			echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
-		fi
-		# echo "$(date) - SYNCED  - ${rcheck} -	${rrmsg}" >> ${FOLDER}/log.repl
-	fi
+	[ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good
 }
\ No newline at end of file
diff --git a/run.sh b/run.sh
index 788161b8..542ba85e 100755
--- a/run.sh
+++ b/run.sh
@@ -36,13 +36,16 @@ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
 			git submodule update --remote >/dev/null 2>&1
 		fi
 		cd ${FOLDER}
-		if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then
+		# if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then
+		if [ "`git log --pretty=%H ...refs/heads/dev^ | head -n 1`" != "`git ls-remote origin -h refs/heads/dev |cut -f1`" ]; then
 			if [ "${2}" != "q" ]; then
 				echo -en "\e[1A";
 				echo -e "\e[0K\r ${idsCL[LightCyan]}Installing updates...${idsCL[Default]}"
 			fi
-			git fetch origin master >/dev/null 2>&1
-			git reset --hard origin/master >/dev/null 2>&1
+			# git fetch origin master >/dev/null 2>&1
+			# git reset --hard origin/master >/dev/null 2>&1
+			git fetch origin dev >/dev/null 2>&1
+			git reset --hard origin/dev >/dev/null 2>&1
 			git reflog expire --expire=now --all >/dev/null 2>&1
 			git repack -ad >/dev/null 2>&1
 			git prune >/dev/null 2>&1