From 0b66a9d4a64d20cef41261a1c82d927235ce769b Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Fri, 26 Jan 2024 09:46:56 -0600
Subject: [PATCH] Update dsmon.sh

---
 dsmon.sh | 82 +++++++++++++++++++++++++++++---------------------------
 1 file changed, 42 insertions(+), 40 deletions(-)

diff --git a/dsmon.sh b/dsmon.sh
index ae75c5e..6072ee4 100755
--- a/dsmon.sh
+++ b/dsmon.sh
@@ -336,6 +336,7 @@ SETUPSSH(){
 	done
 }
 
+
 RUNCMD(){
 	start=`date +%s`
 	
@@ -344,19 +345,40 @@ RUNCMD(){
 	
 	while read hostid hostname hostip hostlimits; do
 		if [ "${hostname}" != "host" ]; then
-			host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
-			host_name[${hostid}]=$hostname
+			host_ip[${hostid}]=$(echo ${hostip} | cut -d ";" -f1)
+			host_name[${hostid}]="${hostname}"
 		fi
 	done <<< $(MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
-
+	
+	ncount=0
 	for hostid in "${!host_ip[@]}"; do
-		checkhost=$(CHECK_HOST ${host_ip[$hostid]})
-		if [ "${checkhost}" != "false" ]; then
-			echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Running command: ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20}${idsCL[Default]}${idsST[Reset]}"
-			ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${host_ip[$hostid]} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20}
-		else
-			echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
+		# checkhost=$(CHECK_HOST ${host_ip[$hostid]})
+		# if [ "${checkhost}" != "false" ]; then
+		# 	echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Running command: ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20}${idsCL[Default]}${idsST[Reset]}"
+		# 	ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${host_ip[$hostid]} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20}
+		# else
+		# 	echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
+		# fi
+		RUNCMD_SUB ${host_ip[$hostid]} ${host_name[$hostid]} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} &
+		((ncount++))
+	done
+	
+	nc_count=0; completed=false
+	until [ "${completed}" == "true" ]; do
+		for nodecmd in /tmp/.dsmon.runcmd.*.done; do
+			HOST=$(grep -oP '(?<=runcmd.).*?(?=.done)' <<< "${nodecmd}")
+			if [ "${HOST}" != "*" ]; then
+				nodecmd_output=$(cat ${nodecmd})
+				[ "${nodecmd_output}" != "" ] && echo -e "${nodecmd_output}\n"
+				rm -f ${nodecmd}
+				((nc_count++))
+			fi
+		done
+		if [ ${nc_count} -eq ${ncount} ]; then
+			completed=true
+			break
 		fi
+		sleep 1s
 	done
 	
 	echo
@@ -365,36 +387,16 @@ RUNCMD(){
 	echo "Script Runtime: ${runtime}"
 	echo
 }
-RUNCMD(){
-	start=`date +%s`
-	
-	declare -A host_ip
-	declare -A host_name
-	
-	while read hostid hostname hostip hostlimits; do
-		if [ "${hostname}" != "host" ]; then
-			host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
-			host_name[${hostid}]=$hostname
-		fi
-	done <<< $(MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
-
-	for hostid in "${!host_ip[@]}"; do
-		# checkhost=$(CHECK_HOST ${host_ip[$hostid]})
-		# if [ "${checkhost}" != "false" ]; then
-			echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Running command: ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20}${idsCL[Default]}${idsST[Reset]}"
-			if ! ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${host_ip[$hostid]} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ; then
-				echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
-			fi
-		# else
-		# 	echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
-		# fi
-	done
-	
-	echo
-	end=`date +%s`
-	runtime=$((end-start))
-	echo "Script Runtime: ${runtime}"
-	echo
+RUNCMD_SUB(){
+	ip=${1}
+	host=${2}
+	tmpfile=/tmp/.dsmon.runcmd.${host// /}
+	touch ${tmpfile}
+	echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host} - Running command: ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22}${idsCL[Default]}${idsST[Reset]}" >> ${tmpfile}.running
+	if ! ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ip} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} > ${tmpfile}.running ; then
+		echo -e "${idsST[Bold]}${idsCL[Red]} ${host} - Timeout${idsCL[Default]}${idsST[Reset]}" > ${tmpfile}.running
+	fi
+	mv ${tmpfile}.running ${tmpfile}.done
 }
 
 UPDATENODES(){
@@ -537,7 +539,7 @@ case ${ACTION} in
 	gettsip) GETTSIP;;
 	gettsip-node) GETTSIP_NODE;;
 	run) RUN ${2};;
-	runcmd) RUNCMD  ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11};;
+	runcmd) RUNCMD  ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21};;
 	update);;
 	updatenodes) UPDATENODES;;
 	*)