#!/usr/bin/env bash

SINGLE_SERVER_SERVICE_CHECK(){
	if [ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check ]; then
		([ "${2^^}" == "START" ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.check)) -ge 1200 ]) && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check
	fi
	if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check ]; then
		touch ${NM_TMPFOLDER}/.singleserver.${1^^}.check
	
		servicestocheck=${1^^}_SINGLESRVR_SERVICES[@]
		dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@]
		hosts=${1^^}_HOSTS[@]
	
		[ "${2^^}" == "START" ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary

		
		# rm -f ${NM_TMPFOLDER}/.singleserver*
			
		if [ ! -f ${NM_TMPFOLDER}/.skip ] && ([[ "$(declare -p ${1^^}_SINGLESRVR_SERVICES)" =~ "declare -a" ]] || [[ "$(declare -p ${1^^}_SINGLESRVR_DOCKERS)" =~ "declare -a" ]] || [ "${1^^}" == "LPD" ] || [ "${1^^}" == "DNS" ] || [ "${1^^}" == "LDNS" ]); then	
			if [ "${1^^}" == "LPD" ] || [ "${1^^}" == "DNS" ] || [ "${1^^}" == "LDNS" ]; then
				dnstest=$(dig +short +tries=3 +time=3 www.google.com @127.0.0.1)
				if [ "${dnstest}" == "" ] || [[ "${dnstest}" == *";;"* ]]; then
					SKIP_SERVER_CHECKS on >/dev/null 2>&1
					completed=false; scanstart=$(date +%s)
					sleep 4s
					until [ "${completed}" == "true" ]; do
						dnstest=$(dig +short +tries=3 +time=3 www.google.com @127.0.0.1)
						if [ "${dnstest}" != "" ] && [[ "${dnstest}" != *";;"* ]]; then
							completed=true
							break
							
						elif [ $(($(date +%s)-scanstart)) -ge 180 ]; then
							SENDNOTICE "${NM_HOSTNAMES[${RUN_NODE_IP}]} DNS Startup Failure 1" "${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP})
Server did not start serving DNS requests within 3mins, starting Pi-Hole repair...." 1
							/usr/local/bin/pihole -r
						
						elif [ $(($(date +%s)-scanstart)) -ge 600 ]; then
							SENDNOTICE "${NM_HOSTNAMES[${RUN_NODE_IP}]} DNS Startup Failure 2" "${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP})
Server did not start serving DNS requests within 10mins you moron...." 1
							killall bash
							exit 1
						fi
						sleep 15s
					done
					/usr/bin/docker restart powerdns_admin >/dev/null 2>&1
					SKIP_SERVER_CHECKS off >/dev/null 2>&1
				fi
				
			elif [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) == *${NM_SINGLESRVR_IP[${1^^}]}* ]]; then
				# if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.primary)) -ge 300 ]; then
				# 	if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
						if [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ]; then
							touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
							SENDNOTICE "'${NM_NODETYPES[${1^^}]}' Node Switch" "Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on other Nodes"
						fi
						echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting SingleServer for '${NM_NODETYPES[${1^^}]}' on ${NM_HOSTNAMES[${RUN_NODE_IP}]} (${RUN_NODE_IP}), stopping on the other Nodes ..." >> ${NM_LOGFILE}
						for stc in "${!servicestocheck}"; do
							[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
							for nip in "${!hosts}"; do
								if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
									${SSHCMD} root@${nip} 'systemctl is-active --quiet '${stc}' && /bin/systemctl stop '${stc} >/dev/null 2>&1
									${SSHCMD} root@${nip} '[ -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary ] && rm -f '${NM_TMPFOLDER}'/.singleserver.'${1^^}'.primary' >/dev/null 2>&1
								fi
							done
						done
						
						#### HERE
						
						for docker in "${!dockerstocheck}"; do
							for nip in "${!hosts}"; do
								if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
									${SSHCMD} root@${nip} '[ "$(docker ps -a -q -f name='${docker}')" ] && docker stop '${docker} >/dev/null 2>&1
									[ "${docker}" == "authelia" ] && ${SSHCMD} root@${nip} '[ "$(docker ps -a -q -f name=auth_redis)" ] && docker stop auth_redis' >/dev/null 2>&1
								fi
							done
						done
						sleep 20s
					# else
						touch ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
					# fi
				
					for stc in "${!servicestocheck}"; do
						if ! systemctl is-active --quiet ${stc}; then
							[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
							/bin/systemctl start ${stc} >/dev/null 2>&1
						fi
					done
					for docker in "${!dockerstocheck}"; do
						if [ ! "$(docker ps -a -q -f name=${docker})" ]; then
							[ "${docker}" == "authelia" ] && docker start auth_redis && sleep 2s >/dev/null 2>&1
							docker start ${docker} >/dev/null 2>&1
						fi
					done
					[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
					echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE}
				# fi
				

			else
				# if [ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] || [ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary)) -ge 300 ]; then
					[ ! -f ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary ] && touch ${NM_TMPFOLDER}/.singleserver.${1^^}.secondary
					for stc in "${!servicestocheck}"; do
						if [ $(systemctl is-active --quiet ${stc}) ]; then
							[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
							/bin/systemctl stop ${stc} & >/dev/null 2>&1
						fi
					done
					for docker in "${!dockerstocheck}"; do
						[ "$(docker ps -a -q -f name=${docker})" ] && docker stop ${docker} & >/dev/null 2>&1
					done
					[ -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary ] && rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.primary
				# fi
			fi
	
			[ "${1^^}" == "WEB" ] && [ "$(systemctl is-active nginx)" != "active" ] && ${NM_SCRIPT} service web restart ${RUN_NODE_IP}
		
		elif [ "${2^^}" == "START" ] && [ -f ${NM_TMPFOLDER}/.skip ]; then
			systemctl stop keepalived >/dev/null 2>&1
		fi
		
		
		rm -f ${NM_TMPFOLDER}/.singleserver.${1^^}.check
	fi
}

SERVICE_MGMT(){
	declare -A SERVICE
	QUIET=0
	while [ $# -gt 0 ]; do
	    case "${1}" in
	        -s|-service) SERVICE['service']=${2,,};;
	        -n|-node) SERVICE['node']=${2^^};;
	        -nt|-nodetype) SERVICE['nodetype']=${2^^};;
	        -a|-action) SERVICE['action']=${2,,};;
	        -q|-quiet) QUIET=1;;
			-h|-help) 
				echo -e "\nUsage: ${idsCL[Yellow]}[nodemgmt or nmg] ${idsCL[LightYellow]}service${idsCL[Default]} {"
				width=35
				printf "%-${width}s- %s\n" "    -s|-service {service}" "Name of service to manage"
				printf "%-${width}s- %s\n" "    -n|-node {node-lan-ip}" "Node to run on"
				printf "%-${width}s- %s\n" "    -nt|-nodetype {nodetype}" "NodeType to run on, or all. Defaults to local nodetype"
				printf "%-${width}s- %s\n" "    -a|-action {action}" "Action to perform; start, stop, restart etc"
				printf "%-${width}s- %s\n" "    -q|-quiet" "Do not output, typically used from cron jobs"
				echo -e "}\n"
				exit 0
				;;
			*)
				if [ "${NM_SERVICE_DESC[${1}]}" != "" ]; then
					SERVICE['service']=${1}
					
				elif [ "${NM_SRVCOPTS[${1}]}" != "" ]; then
					SERVICE['action']=${1}
					
				elif [ "${NM_HOSTNAMES[${1}]}" != "" ]; then
					SERVICE['node']=${1}
					
				elif [ "${1}" == "q" ]; then
					QUIET=1
					
				fi
				;;
	    esac
		shift
	done
	# echo "SERVICE=${SERVICE['service']}"
	# echo "SERVICE DESC=${NM_SERVICE_DESC[${SERVICE['service']}]}"
	# echo "ACTION=${SERVICE['action']}"
	# echo "NODE=${SERVICE['node']}"
	# echo "NODETYPE=${SERVICE['nodetype']}"
	# echo "QUIET=${QUIET}"
	
	if [ ! "${NM_SERVICE_DESC[${SERVICE['service']}]}" ] && [ "${NM_SERVICE_DESC[${SERVICE['service']}]}" != "ALL" ]; then
		echo -e "${idsCL[Red]}"${SERVICE['service']}" is an unknown Service.${idsCL[Default]}"
		exit 1	
	fi
	# [ "${SERVICE['nodetype']}" == "" ] && SERVICE['nodetype']=
	
	if [ "${SERVICE['nodetype']}" == "ALL" ]; then
		# NT_HOSTS=ALL_HOSTS[@]
		NT_HOST=RNIP
		NTS=(${NM_NODE_TYPES[*]})
		
	elif [ "${SERVICE['nodetype']}" != "" ]; then
		if [ "${NM_HOSTS[${SERVICE['nodetype']}]}" != "" ]; then
			NT_HOSTS=${SERVICE['nodetype']}_HOSTS[@]
			NT_HOST=${SERVICE['nodetype']}_HOSTS[0]
			NTS=("${SERVICE['nodetype']}");
		else
			echo -e "${idsCL[Red]}"${SERVICE['nodetype']}" is an unknown NodeType.${idsCL[Default]}"
			exit 1	
		fi
		
	elif [ "${RUN_NODE_TYPE}" != "" ]; then
			NT_HOSTS=${RUN_NODE_TYPE}_HOSTS[@]
			NT_HOST=${RUN_NODE_TYPE}_HOSTS[0]
			NTS=("${RUN_NODE_TYPE}");

	# else
	# 	NTS=(${NM_NODE_TYPES[*]})
	fi
	
	#var=NC_HOSTS[@]
	
	if [ "${SERVICE['service']}" = "glusterd" ]; then
		
		for nip in "${GL_HOSTS[@]}"; do
			if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="${SSHCMD} root@${nip}"
			fi

			[ ${QUIET} -eq 0 ] && echo -en "${NM_SRVCOPTS[${SERVICE['action']}]}ing on '$nip'... ${idsCL[Default]}"
			checkhost=$(CHECK_HOST ${nip})
			if [ "${checkhost}" != "false" ]; then
				if [ ${QUIET} -eq 0 ]; then
					if [[ "enable,disable" = *"${SERVICE['action']}"* ]] || [ "${SERVICE['service']}" = "daemon-reload" ]; then
						echo -e "${idsCL[Green]}Done${idsCL[Default]}"
					elif [ "$(${NCMD} systemctl is-active ${SERVICE['service']} | tr -d $'\r')" = "active" ]; then
						echo -e "${idsCL[Green]}Done${idsCL[Default]}"
					elif [ "${SERVICE['action']}" = "stop" ]; then
						echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
					else
						echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
					fi
				fi
			elif [ ${QUIET} -eq 0 ]; then
				echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
			fi

		done
		
		if [ "${SERVICE['action']}" = "start" ] || [ "${SERVICE['action']}" = "restart" ]; then
			
			echo
			sleep 5s
			
			for nip in "${GL_HOSTS[@]}"; do
				if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
					nip='localhost '
					NCMD=''
				else
					NCMD="${SSHCMD} root@${nip}"
				fi
			
				if [ ${QUIET} -eq 0 ]; then
					echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
				fi
				checkhost=$(CHECK_HOST ${nip})
				if [ "${checkhost}" != "false" ]; then
					#${NCMD} mount -a > /dev/null 2>&1
					if [ ${QUIET} -eq 0 ]; then
						# if [[ "${NCHOSTS}" == *"${nip}"* ]]; then
						if [[ " ${NC_HOSTS[*]} " =~ " ${nip} " ]]; then
							mntpath="/mnt/Nextcloud-Data"
						# elif [[ "${WEBHOSTS}" == *"${nip}"* ]]; then
						elif [[ " ${NM_HOSTS['WEB'][*]} " =~ " ${nip} " ]]; then
							mntpath="/mnt/web-data"
						fi
						${NCMD} "test -e ${mntpath}/mounted"
						if [ $? -eq 0 ]; then
							echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
						else
							echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
						fi
					fi
					
					if [ "$nip" = "10.10.1.43" ]; then
						echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
						mntpath="/mnt/Nextcloud-Data"
						${NCMD} "test -e ${mntpath}/mounted"
						if [ $? -eq 0 ]; then
							echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
						else
							echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
						fi
					fi
					
				elif [ ${QUIET} -eq 0 ]; then
					echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
				fi
			
			done
			
		fi
		
		
	else

		
		# if [ "${RUN_NODE_TYPE}" == "" ]; then
		#
		# fi
		if [ "${NM_SERVICE_DESC[${SERVICE['service']}]}" = "" ]; then
			echo -e "${idsCL[Red]}(${SERVICE['service']}) is not an allowed service.${idsCL[Default]}"
			exit 1	
		fi
		if [ "${NM_SRVCOPTS[${SERVICE['action']}]}" = "" ]; then
			echo -e "${idsCL[Red]}(${SERVICE['action']}) is not an allowed service action.${idsCL[Default]}"
			exit 1
		fi
		if [ ${QUIET} -eq 0 ]; then
			if [ "${SERVICE['action']}" = "start" ] &&  [ "${SERVICE['service']}" = "haproxy" ]; then
				TACT='restart'
			else
				TACT="${SERVICE['action']}"
			fi

			echo
			echo -e "${idsCL[LightGreen]}[[ ${NM_SERVICE_DESC[${SERVICE['service']}]} ${NM_SRVCOPTS[${SERVICE['action']}]}ing ]]${idsCL[Default]}"
			echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
		fi
	
		# echo "HERE: ${RUN_NODE_TYPE} - ${NT_HOST} = ${!NT_HOST}"
		if ([ "${SERVICE['service']}" = "nginx" ] || [ "${SERVICE['service']}" = "apache2" ]) && ([ "${SERVICE['node']}" == "" ] || [ "${SERVICE['node']}" == "ALL" ]); then			
			if [ "${RUN_NODE_TYPE}" != "" ] && [ "${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}" != "" ] && ([ "${SERVICE['action']}" = "restart" ] || [ "${SERVICE['action']}" = "reload" ]); then
				if [ "${SERVICE['node']}" != "ns" ]; then
					if [ ${QUIET} -eq 0 ]; then
						echo -en "${idsCL[LightYellow]}Verifying replication across the nodes... ${idsCL[Default]}"
					fi
					if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then
						PH_CMD=""
					else
						PH_CMD="${SSHCMD} root@${!NT_HOST}"
					fi
					${PH_CMD} touch ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl
					# ${PH_CMD} "echo -e \"Service ${SERVICE['service']} ${SERVICE['action']}\" >> ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl"
					for nip in "${!NT_HOSTS}"; do
						checkhost=$(CHECK_HOST ${nip})
						if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then
							checked=false
							until [ "${checked}" = "true" ]; do
								checked="`${PH_CMD} ${SSHCMD} root@${nip} [ -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl)) -le 90 ] && echo true || echo false`"
							done
						fi
					done
					
					for nip in "${!NT_HOSTS}"; do
						checkhost=$(CHECK_HOST ${nip})
						if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then
							unset checked1 checked2
							until [ "${checked1}" = "true" ]; do
								until [ "${checked2}" = "true" ]; do checked2="`${PH_CMD} [ -f ${NM_REPL_CHECK_LOC[${SERVICE['service']}]}/.test.repl ] && echo true`"; sleep 1s; done
								checked1="`${PH_CMD} [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_CHECK_LOC[${SERVICE['service']}]}/.test.repl)) -le 90 ] && echo true`"
							done
						fi
					done					
					${PH_CMD} rm -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl
				fi
			fi
		elif ([ "${SERVICE['service']}" = "pdns" ] || [ "${SERVICE['service']}" = "pdnsadmin" ] || [ "${SERVICE['service']}" = "headscale" ]) && ([ "${SERVICE['node']}" == "" ] || [ "${SERVICE['node']}" == "ALL" ]); then			
			if [ "${SERVICE['action']}" = "start" ] || [ "${SERVICE['action']}" = "restart" ] || [ "${SERVICE['action']}" = "reload" ]; then
				if [ "${SERVICE['node']}" != "ns" ]; then
					[ ${QUIET} -eq 0 ] && echo -en "${idsCL[LightYellow]}Verifying replication across the nodes... ${idsCL[Default]}"
					if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then
						PH_CMD=""
					else
						PH_CMD="${SSHCMD} root@${!NT_HOST}"
					fi
					${PH_CMD} touch ${NM_REPL_CHECK_LOC[${SERVICE['service']}]}/.test.repl
					for nip in "${!NT_HOSTS}"; do
						checkhost=$(CHECK_HOST ${nip})
						if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then
							unset checked1 checked2
							until [ "${checked1}" = "true" ]; do
								until [ "${checked2}" = "true" ]; do checked2="`${PH_CMD} [ -f ${NM_REPL_CHECK_LOC[${SERVICE['service']}]}/.test.repl ] && echo true`"; sleep 1s; done
								checked1="`${PH_CMD} [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_CHECK_LOC[${SERVICE['service']}]}/.test.repl)) -le 90 ] && echo true`"
							done
						fi
					done
					${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${SERVICE['service']}]}/.test.repl
				fi
			fi
		fi
		if [ ${QUIET} -eq 0 ]; then
			echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
			echo
		fi
		
		if [[ "$(declare -p NTS)" =~ "declare -a" ]]; then
			for NTYPE in "${NTS[@]}"; do
				var1=${NTYPE}_SERVICES_CHECK[@]
				# if [[ "${!var1}" = *"${SERVICE['service']}"* ]]; then
				if [[ " ${!var1} " =~ " ${SERVICE['service']} " ]]; then
					if [ "${SERVICE['node']}" != "" ]; then
						var2=(${SERVICE['node']})
						sethost=true;
					else
						var2=${NTYPE}_HOSTS[@]
						IFS=' '
						var2=(${!var2})
						unset IFS
						sethost=false
					fi
				
					for nip in "${var2[@]}"; do
						# if [[ "${RUN_NODE_IP}" == *"${SERVICE['node']}"* ]]; then GO=true;
						# elif [ ! -z ${SERVICE['node']+x} ] || [ "${SERVICE['node']}" == "q" ]; then GO=true;
						# else GO=false;
						# fi
						# if [ "${GO}" == "true" ]; then
						if [ $sethost == true ]; then
							nodename=${NM_HOSTNAMES[${SERVICE['node']}]}
						else
							nodename="${NM_HOSTNAMES[${nip}]}[${nip}]"
						fi
							if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
								nip='localhost '
								NCMD=''
							else
								NCMD="${SSHCMD} root@${nip}"
							fi
					
							NOGOCHK=true;
							if [ "${SERVICE['service']}" == "gitea" ]; then
								if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *$(GET_AUTHELIA_IP)* ]]; then
									NOGOCHK=false;
								fi
							elif [ "${SERVICE['service']}" == "headscale" ]; then
								if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP['HS']}* ]]; then
									NOGOCHK=false;
								fi
							fi
					
							if [ "${NOGOCHK}" == "true" ]; then
								if [ ${QUIET} -eq 0 ]; then
									echo -en "${NM_SRVCOPTS[${SERVICE['action']}]}ing on ${nodename} ($nip)... ${idsCL[Default]}"
								fi
								checkhost=$(CHECK_HOST ${nip})
								if [ "${checkhost}" != "false" ]; then
								
									if [ "${SERVICE['service']}"= = "offsite-power-check" ]; then
										if [ ${QUIET} -eq 0 ] && [ "${SERVICE['action']}" != "status" ]; then
											if [ "$(${SSHCMD} root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
												echo -e "${idsCL[Green]}Done${idsCL[Default]}"
											else
												echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
											fi
										fi
									else
										if [ "${SERVICE['service']}" = "pdnsadmin" ]; then
											srva="pdnsadmin.socket pdnsadmin.service"
										elif [ "${SERVICE['service']}" = "syncthing" ] && [[ "${NM_SERVICES_CHECK[${NTYPE}]}" = *"syncthing@pdns"* ]]; then
											srva="syncthing syncthing@pdns"
										else
											srva=${SERVICE['service']}
										fi
										if [ "${SERVICE['action']}" != "status" ]; then
											${NCMD} systemctl ${SERVICE['action']} ${srva} >/dev/null 2>&1
										else
											echo
											${NCMD} systemctl ${SERVICE['action']} ${srva}
											echo
										fi
										if [ ${QUIET} -eq 0 ] && [ "${SERVICE['action']}" != "status" ]; then
											SStatus=$(${NCMD} systemctl is-active ${SERVICE['service']} | tr -d $'\r')
											if [[ "enable,disable" = *"${SERVICE['action']}"* ]] || [ "${SERVICE['service']}" = "daemon-reload" ]; then
												echo -e "${idsCL[Green]}Done${idsCL[Default]}"
											elif [[ "${srva}" = *".socket"* ]] && [ "${SStatus}" = "active" ]; then
												echo -e "${idsCL[Green]}Done (& socket service)${idsCL[Default]}"
											elif [[ "${srva}" = *"@pdns"* ]] && [ "${SStatus}" = "active" ]; then
												echo -e "${idsCL[Green]}Done (& @pdns)${idsCL[Default]}"
											elif [ "${SStatus}" = "active" ]; then
												echo -e "${idsCL[Green]}Done${idsCL[Default]}"
											elif [ "${SERVICE['action']}" = "stop" ]; then
												echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
											else
												echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
											fi
										fi
									fi
								
								elif [ ${QUIET} -eq 0 ]; then
									echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
								fi
							fi
						# fi
						# if [ "${SERVICE['service']}" = "gitea" ]; then
						# 	break
						# fi
					done
				fi
			done
			
		else #no-nodetypes, standalone setup
			echo -en "${NM_SRVCOPTS[${SERVICE['action']}]}ing on localhost ... ${idsCL[Default]}"
			if [ "${SERVICE['action']}" != "status" ]; then
				systemctl ${SERVICE['action']} ${srva} >/dev/null 2>&1
			else
				echo
				systemctl ${SERVICE['action']} ${srva}
				echo
			fi
			if [ ${QUIET} -eq 0 ] && [ "${SERVICE['action']}" != "status" ]; then
				if [[ "enable,disable" = *"${SERVICE['action']}"* ]] || [ "${SERVICE['service']}" = "daemon-reload" ]; then
					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
				elif [ "$(systemctl is-active ${SERVICE['service']})" = "active" ]; then
					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
				elif [ "${SERVICE['action']}" = "stop" ]; then
					echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
				else
					echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
				fi
			fi			
		fi
	fi
	
	if [ ${QUIET} -eq 0 ]; then
		echo
		echo -e "${idsCL[Green]}${NM_SERVICE_DESC[${SERVICE['service']}]} has been ${NM_SRVCOPTS[${SERVICE['action']}]}ed${idsCL[Default]}"
		echo
	fi
}