#!/usr/bin/env bash
# DSMon - Disk and Basic System Monitor Script

[ "${3}" != "q" ] && source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc

[ -d /opt/idssys/nodemgmt ] && source /opt/idssys/nodemgmt/defaults.inc

source /opt/idssys/dsmon/config.settings.inc
source /opt/idssys/dsmon/defaults.inc



ACTION="${1}"

RUN(){
	if [ "${SERVERMON_ID}" != "" ]; then

		if [ "${1}" = "hdd" ]; then
			DRIVEINFO=$(df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|localhost|shm|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
			DRIVEINFO=(${DRIVEINFO})
			NUMDRIVES=$((${#DRIVEINFO[@]} / 3))

			DRIVE_INFO=()

			for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
				ii=$((${i}*3))
				ia=${DRIVEINFO[${ii}]}
				ib=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]//M/}/1024")
				ic=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]//M/}/1024")
				ia=${ia//\/dev\/mapper\//}
				ia=${ia//\/dev\//}
			
				if [[ "${ia}" = *"sda"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
					systempartition=true
				elif [ "${ia}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
					systempartition=true
				elif [[ "${ia}" = *"nvme"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
					systempartition=true
				elif [[ "${ia}" = *"mmcblk"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
					systempartition=true
				elif [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
					systempartition=true
				else
					# echo "${ia};${ib};${ic}"
					DRIVE_INFO+=("${ia};${ib};${ic}")
				fi
			
			done
			
			QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','hdd','${DRIVE_INFO[@]}');"
			# echo $QRY
			MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "${QRY}"
		
		elif [ "${1}" = "sysXXX" ]; then
		
			MEMORY=$(free -m | awk 'NR==2{printf "%.2f\t\t", $3*100/$2 }')
			CPU=$(top -bn1 | grep load | awk '{printf "%.2f\t\t\n", $(NF-2)}')
			
			QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','sys','${CPU//		/};${MEMORY}');"
			MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "${QRY}"
		fi
	
		SERVERHOSTNAME=(`hostname`)
		SERVERIP=(`hostname -I`)
		SERVERIPS=''
		for nip in "${SERVERIP[@]}"; do
			if [[ "${nip}" == *"."* ]] && [[ "${nip}" != *"172"* ]] && [[ "${nip}" != *"169"* ]]; then
				if [ "${SERVERIPS}" = "" ]; then
					SERVERIPS=${nip} 
				else
					SERVERIPS=${SERVERIPS}";"${nip} 
				fi
			fi
		done
	
		thresholdlog=()
		for KEY in "${!THRESHOLD[@]}"; do
			thresholdlog+=("${KEY}:${THRESHOLD[$KEY]}")
		done

		QRY="USE servermonitor; UPDATE hosts SET ip='${SERVERIPS// /}',hostname='${SERVERHOSTNAME}',limits='${thresholdlog[@]}' WHERE id='${SERVERMON_ID}';"
		MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "${QRY}"
	

	fi
}

CHECK(){
	rm -f /tmp/dscheck.*
	start=`date +%s`
	
	C_CW=40

	declare -A host_ip
	declare -A host_name
	declare -A host_limits
	declare -A host_ids
	declare -a hostnames_sort
	
	if [ "${1}" == "report" ]; then
		unset idsCL idsBG idsST
		idsCL=('')
		idsBG=('')
		idsST=('')
	fi
	
	if [ "${1}" != "report" ]; then
		MSG="Determining hosts and preparing"
		c=0; cw=${C_CW}; spc=''; spc1=$(expr ${cw} - ${#MSG}); until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
		echo -en "${idsCL[LightCyan]}${MSG}${spc}: "
	fi
	while read hostid hostname hostip hostlimits; do
		if [ "$hostid" != "id" ]; then
			hostip=$(echo $hostip | cut -d ";" -f1)
			# echo "$hostid  - $hostip - $hostname - $hostlimits"
			host_ip[${hostid}]=$hostip
			host_name[${hostid}]=$hostname
			host_limits[${hostid}]=$hostlimits
			host_ids[$hostname]=$hostid
			hostnames_sort+=("$hostname")
		fi
		done <<< $(MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")
	# done <<< $(MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 AND host LIKE '%WAN%'")	
	IFS=$'\n' hostnames_sort=($(sort <<<"${hostnames_sort[*]}")); unset IFS
	[ "${1}" != "report" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"
	
	if [ "${1}" != "report" ]; then
		MSG="Starting jobs to collect host info"
		c=0; cw=${C_CW}; spc=''; spc1=$(expr ${cw} - ${#MSG}); until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
		echo -en "${idsCL[LightCyan]}${MSG}${spc}: "
	fi
	#for hostid in "${!host_ip[@]}"; do
	for hostname in "${hostnames_sort[@]}"; do
		until [ $(ls /tmp/dscheck.*.running 2>/dev/null | wc -l) -lt 6 ]; do tmp=tmp; done
		RUN_CHECK ${hostname} > /tmp/dscheck.${hostname}.results 2>/dev/null &
	done
	[ "${1}" != "report" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
	
	nc_count=0; completed=false; scanstart=$(date +%s)
	until [ "${completed}" == "true" ]; do
		# until [ $(ls /tmp/dscheck.*.done 2>/dev/null | wc -l) -gt 0 ]; do tmp=tmp; done
		for host in /tmp/dscheck.*.done; do
			NTS=$(grep -oP '(?<=dscheck.).*?(?=.done)' <<< "${host}")
			if [ "${NTS}" != "*" ]; then
				hoststatus=$(cat /tmp/dscheck.${NTS}.results)
				[ "${hoststatus}" != "" ] && echo -e "${hoststatus}"
				rm -f /tmp/dscheck.${NTS}.*
				((nc_count++))
			fi
		done
		if [ $(($(date +%s)-scanstart)) -gt 300 ]; then
			SENDNOTICE "DSmon Scan Stuck" "DSmon Check scan has been running for 10mins, klling all bash...." 1
			killall bash
			exit 1
		fi
		if [ ${nc_count} -eq ${#hostnames_sort[@]} ]; then
			completed=true
			break
		fi
		sleep 1s
	done
	
	end=`date +%s`; runtime=$((end-start)); echo -e "\nScript Runtime: ${runtime}"
	echo
}

RUN_CHECK(){
	hostname=${1}
	touch /tmp/dscheck.${hostname}.running
	hostid=${host_ids[$hostname]}
	# if [ "${1}" != "report" ]; then
		echo -e "${idsST[Bold]}"; DIVIDER false lightCyan 85
		echo -e "${idsCL[LightCyan]} ${host_name[$hostid]} - Drive Space Check${idsCL[Default]}"
		DIVIDER false lightCyan 85
	# fi

	checkhost=$(CHECK_HOST ${host_ip[$hostid]})
	if [ "${checkhost}" != "false" ]; then
		declare -A host_limits_tmp
		hostlimits=(${host_limits[${hostid}]})
		for hl in ${hostlimits[@]}; do
			hlname=$(echo $hl | cut -d ":" -f1)
			hllim=$(echo $hl | cut -d ":" -f2)
			host_limits_tmp[$hlname]=$hllim
		done
	
		DRIVEINFO=$(ssh root@${host_ip[$hostid]} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|localhost|shm|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
		DRIVEINFO=(${DRIVEINFO})

		# for x in "${!DRIVEINFO[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO[$x]}" ; done

		NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
		declare -A DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVEINFO_SHORTNAME
		declare -a DRIVES

		for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
			ii=$((${i}*3))
			dname=${DRIVEINFO[${ii}]}
			dname=${dname//\/dev\/mapper\//}
			dname=${dname//\/dev\//}
			dname_short=${dname#*vg-}
			dname_short=${dname_short%*_v}
			[ "$dname_short" = "" ] && dname_short=$dname

			if [[ "${ia}" = *"sda"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
				systempartition=true
			elif [ "${ia}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
				systempartition=true
			elif [[ "${ia}" = *"nvme"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
				systempartition=true
			elif [[ "${ia}" = *"mmcblk"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
				systempartition=true
			elif [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
				systempartition=true
			else
				dtot=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]}/1024/10")
				dfree=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]}/1024/10")
				dfreeper=$(printf "%0.2f" $(jq -n ${dfree}/${dtot}*100))

				DRIVEINFO_TOT[$dname]=$dtot
				DRIVEINFO_FREE[$dname]=$dfree
				DRIVEINFO_FREEPER[$dname]=$dfreeper
				DRIVEINFO_SHORTNAME[$dname]=$dname_short
				DRIVES+=("$dname")
			fi
		done

		IFS=$'\n' DRIVES=($(sort <<<"${DRIVES[*]}")); unset IFS

		# for x in "${!DRIVEINFO_TOT[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_TOT[$x]}" ; done
		# echo
		# for x in "${!DRIVEINFO_FREE[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREE[$x]}" ; done
		# echo
		# for x in "${!DRIVEINFO_FREEPER[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREEPER[$x]}" ; done
		# echo
	
		# if [ "${1}" != "report" ]; then
			c=0; cw=18; spcA=''; spc1=`expr ${cw} - 7`; until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
			c=0; cw=10; spcB=''; spc2=`expr ${cw} - 4`; until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
			c=0; cw=10; spcC=''; spc3=`expr ${cw} - 5`; until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done
			c=0; cw=14; spcD=''; spc4=`expr ${cw} - 3`; until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done
			echo -e "${idsST[Reset]}${idsCL[White]} DRIVE${spcA}  FREE GB${spcB}FREE %${spcC}TOTAL GB${spcD}WARN % / CRIT %${idsCL[Default]}"
			# pf="%-20s %10s %10s %12s %50s\n"
			# printf "${pf}" "${idsCL[White]}DRIVE" "FREE GB" "FREE %" "TOTAL GB" "WARN % / CRIT %${idsCL[Default]}"
		# fi

		for DRIVE in "${DRIVES[@]}"; do
		
			[ "${host_limits_tmp[$DRIVE]}" = "" ] && host_limits_tmp[$DRIVE]=81
		
			WARNING_LEVEL=$(printf "%0.2f" $(jq -n 100-${host_limits_tmp[$DRIVE]}))
			CRITICAL_LEVEL=$(printf "%0.2f" $(jq -n $WARNING_LEVEL-$WARNING_LEVEL/2))
		
			# if [ "${1}" != "report" ]; then
				c=0; cw=18; spcA=''; spc1=`expr ${cw} - ${#DRIVEINFO_SHORTNAME[$DRIVE]}`; until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
				c=0; cw=10; spcB=''; spc2=`expr ${cw} - ${#DRIVEINFO_FREE[$DRIVE]}`; until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
				c=0; cw=10; spcC=''; spc3=`expr ${cw} - ${#DRIVEINFO_FREEPER[$DRIVE]}`; until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done
				c=0; cw=14; spcD=''; spc4=`expr ${cw} - ${#DRIVEINFO_TOT[$DRIVE]}`; until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done
			# fi
		
			if (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${CRITICAL_LEVEL}") )); then
				fs_status='CRITICAL'
				fs_priority=2
				fs_status_color='RedBold'

			elif (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${WARNING_LEVEL}") )); then
				fs_status='Warning'
				fs_priority=1
				fs_status_color='Yellow'
			
			elif (( $(bc <<<"${DRIVEINFO_FREE[$DRIVE]} < 5") )); then
				fs_status='Warning'
				fs_priority=1
				fs_status_color='Magenta'
			else
				fs_status='Normal'
				fs_priority=0
				fs_status_color='Green'
			fi
		
			D_WARNING_LEVEL=$(printf "%0.2f" $(jq -n 100-${WARNING_LEVEL}))
			D_CRITICAL_LEVEL=$(printf "%0.2f" $(jq -n 100-${CRITICAL_LEVEL}))
			# [ "${1}" != "report" ] &&
			echo -e "${idsCL[Cyan]} ${DRIVEINFO_SHORTNAME[$DRIVE]}${spcA}${idsCL[$fs_status_color]}${DRIVEINFO_FREE[$DRIVE]} GB${spcB}${DRIVEINFO_FREEPER[$DRIVE]}%${spcC}${DRIVEINFO_TOT[$DRIVE]} GB${idsST[Reset]}${spcD}${idsCL[Cyan]}( ${idsCL[Yellow]}${D_WARNING_LEVEL}%${idsCL[Cyan]} / ${idsCL[RedBold]}${D_CRITICAL_LEVEL}%${idsST[Reset]}${idsCL[Cyan]} )${idsCL[Default]}"
		
			# [ "${1}" != "report" ] && printf "${pf}" "${idsCL[Cyan]}${DRIVEINFO_SHORTNAME[$DRIVE]}" "${idsCL[$fs_status_color]}${DRIVEINFO_FREE[$DRIVE]} GB" "${DRIVEINFO_FREEPER[$DRIVE]}%" "${DRIVEINFO_TOT[$DRIVE]} GB" "${idsCL[Cyan]}(${idsCL[Default]}${D_WARNING_LEVEL}%${idsCL[Cyan]}/${idsCL[Default]}${D_CRITICAL_LEVEL}%${idsCL[Cyan]})(${idsCL[Yellow]}${D_WARNING_LEVEL}%${idsCL[Cyan]}/${idsCL[Red]}${D_CRITICAL_LEVEL}%${idsCL[Cyan]})${idsCL[Default]}"
		
			if [ "${fs_status}" != "Normal" ] && [ "${1}" == "report" ]; then
				# echo -e "${host_name[$hostid]} - (${host_ip[$hostid]})\n\n${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]}GB out of ${DRIVEINFO_TOT[$DRIVE]}GB Free (${DRIVEINFO_FREEPER[$DRIVE]}%)\n\n$(date)" | mail -s "Free Space ${fs_status}: '${host_name[$hostid]}'" ${ALERT_EMAIL}
				SENDNOTICE "Free Space ${fs_status}: '${host_name[$hostid]}'" "${host_name[$hostid]} - (${host_ip[$hostid]})

	${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]}GB out of ${DRIVEINFO_TOT[$DRIVE]}GB Free (${DRIVEINFO_FREEPER[$DRIVE]})percent" ${fs_priority}
			fi
		
		done
	
		unset DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVEINFO_SHORTNAME DRIVES DRIVEINFO host_limits_tmp
	
	else
		# [ "${1}" != "report" ] &&
		echo -e "${idsCL[Yellow]} ${host_name[$hostid]} is down${idsCL[Default]}"; echo
	fi
	mv /tmp/dscheck.${hostname}.running /tmp/dscheck.${hostname}.done
}

SETUPSSH(){
	declare -A host_ip
	declare -A host_name
		
	while read hostid hostname hostip hostlimits; do
		if [ "$hostid" != "id" ]; then
			host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
			host_name[${hostid}]=$hostname
		fi
	done <<< $(MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")
	
	for hostid in "${!host_ip[@]}"; do
		checkhost=$(CHECK_HOST ${host_ip[$hostid]})
		if [ "${checkhost}" != "false" ]; then
			echo -en "\n${idsCL[LightCyan]}${host_name[$hostid]} - Checking for Public SSH Key ... ${idsCL[Default]}"
			if [ "$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1)" == "ok" ]; then
				echo -e "${idsCL[Green]}Already Installed${idsCL[Default]}"
			else
				echo -e "${idsCL[Yellow]}Copying to Host ...${idsCL[Default]}"
				ssh-copy-id -f root@${host_ip[$hostid]}
			fi
		fi
	done
}


RUNCMD(){
	start=`date +%s`
	rm -f /tmp/.dsmon.runcmd.*
	echo -e "\n${idsCL[White]}The following command will be run on all nodes:"
	echo -e "${idsST[Bold]}'${idsCL[LightCyan]}$(echo "${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20}" | awk '{$1=$1};1')${idsCL[White]}'${idsCL[Default]}${idsST[Reset]}\n"
	ncount=0
	
	# while read hostid hostname hostip hostlimits; do
	# 	if [ "${hostname}" != "host" ]; then
	# 		ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${hostip} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22}
	# 	fi
	# done <<< $(MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
	
	
	while read hostid hostname hostip hostlimits; do
		if [ "${hostname}" != "host" ]; then
			RUNCMD_SUB $(echo ${hostip} | cut -d ";" -f1) ${hostname} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} &
			((ncount++))
		fi
	done <<< $(MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")

	nc_count=0; lcount=0; completed=false
	until [ "${completed}" == "true" ]; do
		for nodecmd in /tmp/.dsmon.runcmd.*.done; do
			# HOST=$(grep -oP '(?<=runcmd.).*?(?=.done)' <<< "${nodecmd}")
			# if [ "${HOST}" != "*" ]; then
			# if [[ "${nodecmd}" != *"*"* ]]; then
			# 	nodecmd_output=$(cat ${nodecmd})
			# 	[ "${nodecmd_output}" != "" ] && echo -e "${nodecmd_output}\n"
			# fi
			# rm -f ${nodecmd}

			if [[ "${nodecmd}" != *"*"* ]]; then
				nodecmd_output=$(cat ${nodecmd})
				[ "${nodecmd_output}" != "" ] && echo -e "${nodecmd_output}\n"
				rm -f ${nodecmd}
				((nc_count++))
			fi
		done
		if [ ${nc_count} -eq ${ncount} ]; then
			echo -e "${idsCL[LightGreen]}Complete, the command was ran on all nodes\n"
			completed=true
			break
		fi
		[ ${lcount} -ne ${nc_count} ] && echo -e "${idsCL[Yellow]}${nc_count} of ${ncount} completed ...\n"
		lcount=${nc_count}

		sleep 2s
	done
	
	echo
	end=`date +%s`
	runtime=$((end-start))
	echo "Script Runtime: ${runtime}"
	echo
}
RUNCMD_SUB(){
	ip=${1}
	host=${2}
	tmpfile=/tmp/.dsmon.runcmd.${host// /-}
	touch ${tmpfile}.running > /dev/null 2>&1
	echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host}${idsCL[Default]}${idsST[Reset]}" >> ${tmpfile}.running
	if ssh_output=$(ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ip} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22}) ; then
		echo -e "${ssh_output}" >> ${tmpfile}.running
	else
		echo -e "${idsCL[Red]} ${host} - Timeout${idsCL[Default]}" >> ${tmpfile}.running
	fi
	mv ${tmpfile}.running ${tmpfile}.done > /dev/null 2>&1
}
RUNCMDSLOW(){
	start=`date +%s`
	echo -e "\n${idsCL[White]}The following command will be run on all nodes:"
	echo -e "${idsST[Bold]}'${idsCL[LightCyan]}$(echo "${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20}" | awk '{$1=$1};1')${idsCL[White]}'${idsCL[Default]}${idsST[Reset]}\n"
	
	while read hostid hostname hostip hostlimits; do
		if [ "${hostname}" != "host" ]; then
			echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${hostname} - $(echo ${hostip} | cut -d ";" -f1)${idsCL[Default]}${idsST[Reset]}"
			ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@$(echo ${hostip} | cut -d ";" -f1) ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} </dev/null
			echo
		fi
	done <<< $(MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
	
	echo -e "${idsCL[LightGreen]}Complete, the command was ran on all nodes\n"
	
	echo
	end=`date +%s`
	runtime=$((end-start))
	echo "Script Runtime: ${runtime}"
	echo
}

UPDATENODES(){
	start=`date +%s`
	
	declare -A host_ip
	declare -A host_name
	
	while read hostid hostname hostip hostlimits; do
		if [ "${hostname}" != "host" ]; then
			echo -e "${idsCL[LightCyan]}Updating DSMON on '${hostname}'${idsCL[Default]}"
			ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@$(echo $hostip | cut -d ";" -f1) "dsmon update" </dev/null
			echo
		fi
	done <<< $(MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
	
	echo
	end=`date +%s`
	runtime=$((end-start))
	echo "Script Runtime: ${runtime}"
	echo

}

GETTSIP_NODE(){
	TSI=$(/sbin/ip link | grep tailscale0)
	if [ ${#TSI} != 0 ]; then
		TSIP=$(/sbin/ip -o -4 addr list tailscale0 | awk '{print $4}' | cut -d/ -f1)
		hostname=$(hostname -f); hostname=${hostname,,}
		([ "${hostname}" == "" ] || [[ "${hostname}" != *".scity"* ]]) && hostname=$(hostname) && hostname="${hostname,,}.ts.scity.us" || hostname=${hostname/.scity/.ts.scity}
		echo "Tailscale running, TSIP: ${TSIP}"
		if [ "${NM_PDNS_USER}" != "" ]; then
			echo -en "Updating DNS for '${hostname}' ... "
			dyndns=$(/usr/bin/curl -s "https://${NM_PDNS_USER}:${NM_PDNS_PASS}@ldns.scity.us/nic/update?hostname=${hostname}&myip=${TSIP}")
			echo "Done (${dyndns})"
		fi
	else
		echo "Not running Tailscale"
	fi
}

GETCRONTABS(){
	start=`date +%s`
	rtpsswd=${1}
	declare -A host_ip
	declare -A host_name
	
	[ ! -d ${DS_CRONTAB_FOLDER} ] && mkdir ${DS_CRONTAB_FOLDER} && chmod -Rf 0777 ${DS_CRONTAB_FOLDER}
	
	while read hostid hostname hostip hostlimits; do
		if [ "$hostid" != "id" ]; then
			host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
			host_name[${hostid}]=$hostname
		fi
	done <<< $(MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
	
	if [ "${rtpsswd}" == "" ]; then
		read -p "Enter the password for the 'nm_crontab_user' account on MySQL-Manager: " rtpsswd
		echo
	fi
	
	for hostid in "${!host_ip[@]}"; do
		# echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - ${host_ip[$hostid]}${idsCL[Default]}${idsST[Reset]}"
		# checkhost=$(CHECK_HOST ${host_ip[$hostid]})
		# if [ "${checkhost}" != "false" ]; then
			# echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Gathering Crontabs ... ${idsCL[Default]}${idsST[Reset]}"
			ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${host_ip[$hostid]} dsmon get-crontab ${rtpsswd}
		# else
		# 	echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
		# fi
	done
	
	echo
	end=`date +%s`
	runtime=$((end-start))
	echo "Script Runtime: ${runtime}"
	echo
	
}
GETCRONTAB(){
	if [ "${1}" != "" ]; then
		if ! command -v sshpass > /dev/null; then
			apt install -y sshpass > /dev/null 2>&1
		fi
 		if [ "${nmip}" != "" ]; then
			while read host hostname; do
				if [ "${host}" != "host" ] && [ "${host}" != "" ]; then
					# echo -e "${SERVERMON_ID}) ${host} : ${hostname}\nmysqlip=${mysqlip}\n"
					crontab -l | sshpass -p${1} ssh -tq -o ConnectTimeout=3 -o ConnectionAttempts=1 -o 'StrictHostKeyChecking no' nm_crontab_user@${nmip} "cat > ${DS_CRONTAB_FOLDER}/${host}.crontab"
				fi
			done <<< $(MYSQL_PWD="sysmoninsert" mariadb --skip-ssl -h ${mysqlip}  -P 3307 -u sysmoninsert -e "SELECT host,hostname FROM servermonitor.hosts WHERE id=${SERVERMON_ID}")
		fi
	
		# [ "${nmip}" != "" ] && crontab -l | sshpass -p${1} ssh -o 'StrictHostKeyChecking no' nm_crontab_user@${nmip} "cat > ${DS_CRONTAB_FOLDER}/${hostname}.crontab"
	
	else
		echo "No user password specified"
	fi
}
FIXCRONTAB(){
	crontab -l | grep -v 'dsmon hdd'  | crontab -
	crontab -l | grep -v 'dsmon sys'  | crontab -
	(crontab -l ; echo "$(($RANDOM % 59)) */1 * * * /usr/local/bin/dsmon hdd >/dev/null 2>&1")| crontab -
	(crontab -l ; echo "*/5+$(($RANDOM % 4)) * * * * /usr/local/bin/dsmon sys >/dev/null 2>&1")| crontab -
	crontab -l | grep "dsmon"
}

# case ${ACTION} in
# 	check) CHECK ${2};;
# 	setupssh) SETUPSSH;;
# 	get-crontabs) GETCRONTABS ${2};;
# 	get-crontab) GETCRONTAB ${2};;
# 	fix-crontab) FIXCRONTAB;;
# 	gettsip) GETTSIP;;
# 	gettsip-node) GETTSIP_NODE;;
# 	run) RUN ${2};;
# 	runcmd) RUNCMD  ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11};;
# 	update);;
# 	*) RUN ${ACTION};;
# esac

if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
	echo
	DIVIDER . lightGreen 70
	msg="| Disk Space Monitor (DSMon) ${idsCL[Default]}                 ${idsCL[DarkGray]}(ver-${VERS})"
	c=0; spc=''; spc1=`expr 81 - ${#msg}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
	echo -e "${idsCL[LightGreen]}${msg}${spc}${idsCL[LightGreen]}|${idsCL[Default]}"
	DIVIDER . lightGreen 70
fi
case ${ACTION} in
	check) CHECK ${2};;
	setupssh) SETUPSSH;;
	get-crontabs) GETCRONTABS ${2};;
	get-crontab) GETCRONTAB ${2};;
	fix-crontab) FIXCRONTAB;;
	gettsip) GETTSIP;;
	gettsip-node) GETTSIP_NODE;;
	run) RUN ${2};;
	runcmd) RUNCMD ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21};;
	runcmdslow) RUNCMDSLOW ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21};;
	update);;
	updatenodes) UPDATENODES;;
	*)
		if [ "${ACTION}" == "hdd" ] || [ "${ACTION}" == "sys" ]; then
			RUN ${ACTION}
		else
			echo -e "${idsCL[White]} Usage: ${idsCL[LightCyan]}dsmon {option}"
			echo
			echo -e "   ${idsCL[Yellow]}check                                         ${idsCL[Default]}--> Run a Disk Space check for all hosts"
			echo -e "   ${idsCL[Yellow]}setupssh                                      ${idsCL[Default]}--> Configure direct SSH access to all hosts"
			echo -e "   ${idsCL[Yellow]}get-crontabs                                  ${idsCL[Default]}--> Get root crontabs from all hosts"
			echo -e "   ${idsCL[Yellow]}gettsip                                       ${idsCL[Default]}--> Check Sensors"
			echo -e "   ${idsCL[Yellow]}gettsip-node                                  ${idsCL[Default]}--> Check Sensors"
			echo -e "   ${idsCL[Yellow]}run {type}                                    ${idsCL[Default]}--> Monitor Check to run"
			echo -e "   ${idsCL[LightYellow]}  type = { hdd / sys }                        ${idsCL[Default]}--> Disk Space or System Resources"
			echo -e "   ${idsCL[Yellow]}runcmd                                        ${idsCL[Default]}--> Run command on all hosts"
			echo -e "   ${idsCL[Yellow]}update                                        ${idsCL[Default]}--> Update DSMon"
			echo
		fi
	;;
esac

exit 0