#!/usr/bin/env bash
# NodeMgmt - Node Management Scripts

action="$1"

source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc

source /opt/idssys/nodemgmt/conf/settings.conf
source /opt/idssys/nodemgmt/defaults.inc

source /opt/idssys/nodemgmt/inc/status.inc
source /opt/idssys/nodemgmt/inc/certs.inc
source /opt/idssys/nodemgmt/inc/sites.inc
source /opt/idssys/nodemgmt/inc/services.inc

if [ "${1}" == "report" ] || [ "${2}" == "report" ] || [ "${3}" == "report" ]; then
	unset idsCL idsBG idsST
	idsCL=('')
	idsBG=('')
	idsST=('')
fi


BACKUP(){
	BACKUP_FOLDER=/opt/idssys/backups/node-backup/$(date +%m-%d-%y)/$(date +%H-%M-%S)
	#BACKUP_FOLDER=/opt/idssys/backups/node-backups/${NODE_HOSTNAME} #/`date +%Y-%m-%d`
	#BACKUP_TIMEFOLDER=${BACKUP_DAYFOLDER}/`date +%H-%M`
	# if [ "${action}" != "backup" ]; then
		echo -e "${idsCL[Green]}Backing up Node Settings and Files...${idsCL[Default]}"
		DIVIDER true
	# fi
	if [ ! -d ${BACKUP_FOLDER} ]; then
		mkdir -p ${BACKUP_FOLDER}
	fi
	for item in "${!NM_BACKUP_ITEMS[@]}"; do
		c=0; spc=''; spc1=`expr 20 - ${#item}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
		echo -en "Backing up '${item}'${spc}"
		if [ -d ${NM_BACKUP_ITEMS[$item]} ]; then
			tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${NM_BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git' >/dev/null 2>&1
			echo -e "${idsCL[Green]}Done${idsCL[Default]}"
		else
			echo -e "${idsCL[Yellow]}Not found${idsCL[Default]}"
		fi
	done
	touch ${NM_LOGFOLDER}/backup.lastrun
	# if [ "${action}" != "backup" ]; then
		echo 
		DIVIDER true
	# fi
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		ENTER2CONTINUE
	fi
}



SET-PERMISSIONS(){
	if [ "${1}" = "gitea" ]; then
		nid=1
		for nip in "${NM_HOSTS['WEB'][@]}"; do
			if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			if [ "${2}" != "q" ]; then
				echo -en "${idsCL[Yellow]}Resetting folder permissions for Gitea on Webserver-Node${nid} (${nip})... ${idsCL[Default]}"
			fi
			checkhost=$(CHECK_HOST ${nip})
			if [ "${checkhost}" != "false" ]; then
				${NCMD} chown -R git:git /var/lib/gitea
				${NCMD} chmod -R 750 /var/lib/gitea
				${NCMD} chown -R git:git /home/git
				${NCMD} chmod -R 750 /home/git
				if [ "${2}" != "q" ]; then
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
				fi
			elif [ "${2}" != "q" ]; then
				echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
			fi
			nid=`expr $nid + 1`
		done
	elif [ "${1}" = "pdnsadmin" ]; then
		nid=1
		for nip in "${NM_HOSTS['WEB'][@]}"; do
			if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			if [ "${2}" != "q" ]; then
				echo -en "${idsCL[Yellow]}Resetting folder permissions for PowerDNS-Admin on Webserver-Node${nid} (${nip})... ${idsCL[Default]}"
			fi
			checkhost=$(CHECK_HOST ${nip})
			if [ "${checkhost}" != "false" ]; then
				${NCMD} chown -R www-data:www-data /opt/powerdns-admin
				${NCMD} chmod -R 775 /opt/powerdns-admin
				if [ "${2}" != "q" ]; then
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
				fi
			elif [ "${2}" != "q" ]; then
				echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
			fi
			nid=`expr $nid + 1`
		done
	elif [ "${1}" = "nextcloud" ]; then
		nip=${NC_HOSTS[0]}
		if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
			NCMD=''
		else
			NCMD="ssh root@${nip}"
		fi
		if [ "${2}" != "q" ]; then
			echo -en "${idsCL[Yellow]}Resetting folder permissions for Nextcloud on Nextcloud-Node1 (${nip})... ${idsCL[Default]}"
		fi
		checkhost=$(CHECK_HOST ${nip})
		if [ "${checkhost}" != "false" ]; then
			${NCMD} chown -R www-data:www-data /mnt/Nextcloud-Data
			${NCMD} find /mnt/Nextcloud-Data/ -type d -exec chmod 770 {} \;
			${NCMD} find /mnt/Nextcloud-Data/ -type f -exec chmod 660 {} \;
			if [ "${2}" != "q" ]; then
				echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
			fi
		elif [ "${2}" != "q" ]; then
			echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
		fi
	else
		DIVIDER
		if [ "${1}" != "" ]; then
			NEW_SITE=${1}
			echo -e "${idsCL[Yellow]}Setting new site folder permissions for (${NM_WWWPATH}/${NEW_SITE})${idsCL[Default]}"
		fi
		if [ "${NEW_SITE}" = "xxx" ]; then
			nid=1
			for nip in "${MYSQL_HOSTS[@]}"; do
				checkhost=$(CHECK_HOST ${nip})
				if [ "${checkhost}" != "false" ]; then
					if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
						nip='localhost '
						NCMD=''
					else
						NCMD="ssh root@${nip}"
					fi
					echo -e "${idsCL[Yellow]}Setting folder permissions for MySQL-Node${nid} (${nip})${idsCL[Default]}"
					echo -en "${idsCL[Cyan]}LetsEncrypt Certs    ${idsCL[Default]}"
					${NCMD} chown -R root:le ${NM_CERTPATH}
					${NCMD} chmod -R 6775 ${NM_CERTPATH}
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
					echo 
				fi
				nid=`expr $nid + 1`
			done
		fi
		nid=1
		[ "${NM_WWWPATHS['WEB']}" != "" ] && WWWPATH=${NM_WWWPATHS['WEB']} || WWWPATH=/var/www
		for nip in "${NM_HOSTS['WEB'][@]}"; do
			checkhost=$(CHECK_HOST ${nip})
			if [ "${checkhost}" != "false" ]; then
				if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
					nip='localhost '
					NCMD=''
				else
					NCMD="ssh root@${nip}"
				fi
				if [ "${NEW_SITE}" != "" ]; then
					echo -en "${idsCL[Cyan]}Webserver-Node${nid} (${nip})    ${idsCL[Default]}"
					${NCMD} chown -R www-data:www-data ${WWWPATH}/$NEW_SITE
					${NCMD} chmod -R 7775 ${WWWPATH}/$NEW_SITE
					${NCMD} chown -R www-data:www-data /var/lib/php/sessions
					${NCMD} chmod -R 7777 /var/lib/php/sessions
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
				else
					echo -e "${idsCL[Yellow]}Setting folder permissions for Webserver-Node${nid} (${nip})${idsCL[Default]}"
					echo -en "${idsCL[Cyan]}Webserver folders    ${idsCL[Default]}"
					${NCMD} chown -R www-data:www-data ${WWWPATH}
					${NCMD} chmod -R 7775 ${WWWPATH}
					${NCMD} chown -R www-data:www-data /var/lib/php/sessions
					${NCMD} chmod -R 7777 /var/lib/php/sessions
					echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
		        	echo -en "${idsCL[Cyan]}Gitea Folder         ${idsCL[Default]}"
					${NCMD} chown -R git:git /var/lib/gitea
					${NCMD} chmod -R 750 /var/lib/gitea
					${NCMD} chown -R git:git /home/git
					${NCMD} chmod -R 750 /home/git
					echo -e "${idsCL[Green]}Complete${idsCL[Def ault]}"
					echo 
				fi
			fi
			nid=`expr $nid + 1`
		done
		if [ "${NEW_SITE}" != "" ]; then echo ; fi
		if [ -z $action ] || [ "${action}" = "gui" ]; then
			ENTER2CONTINUE
		fi
	fi
}

HAPROXY-CONFIG() {
	echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}"
	DIVIDER false yellow
	# if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then
	if [[ " ${MYSQL_SERVICES_CHECK[*]} " =~ " haproxy " ]]; then
		nid=1
		for nip in "${MYSQL_HOSTS[@]}"; do
			if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on MySQL-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
		echo
	fi
	# if [[ "${WEBSERVICES_CHECK}" = *"haproxy"* ]]; then
	if [[ " ${WEB_SERVICES_CHECK[*]} " =~ " haproxy " ]]; then
		nid=1
		for nip in "${NM_HOSTS['WEB'][@]}"; do
			if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
	fi
	# if [[ "${LBSERVICES_CHECK}" = *"haproxy"* ]]; then
	if [[ " ${LB_SERVICES_CHECK[*]} " =~ " haproxy " ]]; then
		nid=1
		for nip in "${LB_HOSTS[@]}"; do
			if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on LB-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
	fi
	SERVICE haproxy reload
	echo 
	echo -e "${idsCL[Green]}HAProxy Config Files have been created.${idsCL[Default]}"
	echo 
}


KEEPALIVE-CONFIG() {
	echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"
	DIVIDER false yellow
	# if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then
	if [[ " ${MYSQL_SERVICES_CHECK[*]} " =~ " keepalived " ]]; then
		nid=1
		for nip in "${MYSQL_HOSTS[@]}"; do
			if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on MySQL-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/keepalived/keepalived.conf"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
		echo
	fi
	# if [[ "${WEBSERVICES_CHECK}" = *"keepalive"* ]]; then
	if [[ " ${WEB_SERVICES_CHECK[*]} " =~ " keepalived " ]]; then
		nid=1
		for nip in "${NM_HOSTS['WEB'][@]}"; do
			if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
				nip='localhost '
				NCMD=''
			else
				NCMD="ssh root@${nip}"
			fi
			echo -en "Creating File on Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
		
			${NCMD} "nodemgmt update q"
			${NCMD} "rm -f /etc/keepalived/keepalived.conf"
			${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
		
			echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
			nid=`expr $nid + 1`
		done
	fi
	SERVICE keepalived reload
	echo 
	echo -e "${idsCL[Green]}Keepalive Config Files have been created.${idsCL[Default]}"
	echo 
}

NODEUPDATE() {
	start=`date +%s`
	if [ "${1}" = "report" ]; then
		unset idsCL idsBG idsST
		idsCL=('')
		idsBG=('')
		idsST=('')
	else
		echo
	fi
	declare -i cw; declare -i spc1; declare -i c
	echo -e "${idsCL[LightGreen]}Updating All Nodes${idsCL[Default]}"
	echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
	nmip=${NM_NODEMANAGER}
	if [[ ${RUN_NODE_IP} == *"${nmip}"* ]]; then
		nmip='localhost'
	fi
	
	if [ "${2}" != "sync" ]; then
		c=0; cw=24; spca=''; spc1=${cw}-7; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
		echo -en "Updating Node-Manager[$nmip]${spca}  ~  ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
		ssh -tq root@${nmip} "nodemgmt update q"
		
		# echo -en "  ~  ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
		# ssh -tq root@${nmip} "runup runup" >/dev/null 2>&1
		# echo -e "${idsCL[Green]}Updated${idsCL[Default]}"
	fi
	
	echo
	
	for NTYPE in "${NM_NODE_TYPES[@]}"; do
		nid=1
		var=${NTYPE}_HOSTS[@]
		for nip in "${!var}"; do
			if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
				nip='localhost  '
			fi
			NCMD="ssh -tq root@${nip}"
			NODE=${NM_HOSTNAMES[${nip}]}[${nip}]
			c=0; cw=40; spca=''; spc1=${cw}-${#NODE}; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
			echo -en "Updating ${NODE}${spca}"

			if [ "${2}" != "sync" ]; then
				echo -en "  ~  ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
				if ! ${NCMD} command -v nodemgmt > /dev/null; then
				    ${NCMD} "bash <(curl -sL https://go.scity.us/install-nodemgmt)" >/dev/null 2>&1
					ssh -tq root@${nmip} rsync -az ${NM_FOLDER}/conf/defaults.local.inc ${NM_FOLDER}/conf/ssl-domain-checks.conf root@${nip}:${NM_FOLDER}/conf/ & >/dev/null 2>&1
					echo -e "${idsCL[Green]}Installed & Synced${idsCL[Default]}"
				else
				    ${NCMD} "nodemgmt update q"
				fi

				# echo -en "  ~  ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
				# if ! ${NCMD} command -v runup > /dev/null; then
				#     ${NCMD} bash <(curl -sL https://go.scity.us/install-linupx) >/dev/null 2>&1
				# else
				#     ${NCMD} "runup runup" >/dev/null 2>&1
				# fi
				# echo -e "${idsCL[Green]}Updated${idsCL[Default]}"

			else
				nmdiv=$(ssh -tq root@${nmip} ls -l --time-style=+%s ${NM_FOLDER}/conf/defaults.local.inc  | cut -d ' ' -f 6)
				nmsdc=$(ssh -tq root@${nmip} ls -l --time-style=+%s ${NM_FOLDER}/conf/ssl-domain-checks.conf  | cut -d ' ' -f 6)
				
				echo -en "  ~  ${idsCL[LightCyan]}Sync Shared Nodemgmt Defaults${idsCL[Default]} : "
				if ssh -tq root@${nip} [[ ! -f ${NM_FOLDER}/conf/defaults.local.inc ]] || 
				ssh -tq root@${nip} [[ ! -f ${NM_FOLDER}/conf/ssl-domain-checks.conf ]] || 
				[ ${nmdiv} -ne $(ssh -tq root@${nip} ls -l --time-style=+%s ${NM_FOLDER}/conf/defaults.local.inc  | cut -d ' ' -f 6) ] || 
				[ ${nmsdc} -ne $(ssh -tq root@${nip} ls -l --time-style=+%s ${NM_FOLDER}/conf/ssl-domain-checks.conf  | cut -d ' ' -f 6) ]; then
					ssh -tq root@${nmip} rsync -az ${NM_FOLDER}/conf/defaults.local.inc ${NM_FOLDER}/conf/ssl-domain-checks.conf root@${nip}:${NM_FOLDER}/conf/ & >/dev/null 2>&1
					echo -e "${idsCL[Green]}Files Synced${idsCL[Default]}"
				else
					echo -e "${idsCL[Green]}Files Already Synced${idsCL[Default]}"
				fi

			fi

			nid=`expr $nid + 1`
		done
		echo
	done
	
	
	echo -e "${idsCL[Green]}All Nodes have been Updated.${idsCL[Default]}"
	echo
	echo
	end=`date +%s`
	runtime=$((end-start))
	echo "runtime: ${runtime}"
}

VCENTER-SSL(){
	VCHOSTNAME='vcenter.scity.us'
	BASECERTDIR="${NM_CERTPATH}/live"
	# BASECERTDIR="/opt/nginx-proxy/ssl"
	CERTDIR="${BASECERTDIR}/${VCHOSTNAME}"
	SERVER="https://${VCHOSTNAME}"
	CRED="administrator@scity.vs:$(pass iDSVC)"	
	VCUSER="administrator@scity.vs"
	CURRENTLIVE=/etc/vmware-rhttpproxy/ssl/rui.crt
	
	CERT=/root/.acme.sh/$VCHOSTNAME/$VCHOSTNAME.cer
	KEY=/root/.acme.sh/$VCHOSTNAME/$VCHOSTNAME.key
	CHAIN=/root/.acme.sh/$VCHOSTNAME/fullchain.cer
	
	echo -en "${idsCL[LightCyan]}Checking days left on vCenter cert... ${idsCL[Default]}"
	VCCERTDAYS=$(${NM_FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N)
	VCCERTDAYS=${VCCERTDAYS#*=}
	
	if [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" != "force" ]; then
		echo -e "${idsCL[Green]}${VCCERTDAYS} days left, Certificate is still valid, no noeed to update${idsCL[Default]}"
		echo
		exit 0
	elif [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" == "force" ]; then
		echo -e "${idsCL[Yellow]}${VCCERTDAYS} days left, forcing certificate update${idsCL[Default]}"
		echo
	else
		echo -e "${idsCL[Yellow]}${VCCERTDAYS} days left, Certificate needs to be updated${idsCL[Default]}"
		echo 
	fi
	
	LIVEMD5=$(ssh root@$VCHOSTNAME "md5sum $CURRENTLIVE | cut -d\  -f1")
	CURRENTMD5=$(md5sum $CERTDIR/cert.pem | cut -d\  -f1)
	if [ "$LIVEMD5" == "$CURRENTMD5" ] && [ "${1}" != "force" ]; then
		echo -e "${idsCL[Yellow]}Certificates remains the same, no newer certificates exist${idsCL[Default]}"
		echo
		exit 0
	fi
	
	echo -e "${idsCL[Green]}Updating certificates on vCenter... ${idsCL[Default]}"
	echo -e "${idsCL[LightCyan]}This process make take up to 10mins${idsCL[Default]}"
	echo
	
	# eval $(awk '{ print "export " $1 }' /etc/sysconfig/vmware-environment)

	scp $BASECERTDIR/$VCHOSTNAME/cert.pem root@${VCHOSTNAME}:$CERT
	scp $BASECERTDIR/$VCHOSTNAME/privkey.pem root@${VCHOSTNAME}:$KEY
	scp $BASECERTDIR/$VCHOSTNAME/fullchain.pem root@${VCHOSTNAME}:$CHAIN
	
	ssh root@$VCHOSTNAME "(printf '1\n%s\n' '$VCUSER'; sleep 1; printf '%s\n' '$(pass iDSVC)'; sleep 1; printf '2\n'; sleep 1; printf '%s\n%s\n%s\ny\n\n' '$CERT' '$KEY' '$CHAIN') | setsid /usr/lib/vmware-vmca/bin/certificate-manager"
	
	
	
	# SESSION_ID=$(curl -s -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' --header 'vmware-use-header-authn: test' --header 'vmware-api-session-id: null' -u $CRED "$SERVER/rest/com/vmware/cis/session" --insecure | python3 -c "import sys, json; print(json.load(sys.stdin)['value'])")
	# PRIVKEY=$(awk -v ORS='\\n' '1' "$CERTDIR/privkey.pem")
	# CERT=$(awk -v ORS='\\n' '1' "$CERTDIR/fullcert.pem")
	# REQUEST_BODY="{ \"spec\" : { \"cert\" : \"$CERT\", \"key\" : \"$PRIVKEY\"  } }"
	# curl --insecure -X PUT "$SERVER/rest/vcenter/certificate-management/vcenter/tls" \
	#                 -H "vmware-api-session-id: $SESSION_ID" \
	#                 -H "Content-type: application/json" \
	#                 -d "$REQUEST_BODY"
	# curl --insecure -X DELETE "$SERVER/rest/com/vmware/cis/session" -H "vmware-api-session-id: $SESSION_ID"
	
	# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
	# echo
	
	# read -n 1 -s -p "Press any key to continue"
	
	# echo -en "${idsCL[LightCyan]}Stopping vCenter services... ${idsCL[Default]}"
	# ssh -i ~/.ssh/id_rsa root@vcenter.scity.us 'service-control --stop --all' >/dev/null 2>&1
	# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
	#
	# echo -en "${idsCL[LightCyan]}Starting vCenter services... ${idsCL[Default]}"
	# ssh -i ~/.ssh/id_rsa root@vcenter.scity.us 'service-control --start --all' >/dev/null 2>&1
	# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
	
	echo
	echo -e "${idsCL[Green]}The vCenter certifcate has been updated${idsCL[Default]}"
	echo -e "${idsCL[LightCyan]}Don't forget to re-scan the vCenter connection in Veeam${idsCL[Default]}"
	echo
}
ADD_LOGROTATE_CRONTAB(){
	if ! crontab -l | grep -q "${NM_FOLDER}/tmp-logrotate"; then
	  (crontab -l ; echo "0 */1 * * * logrotate -f ${NM_FOLDER}/tmp-logrotate") >/dev/null 2>&1 | crontab -
	fi
	/bin/chmod 0644 ${NM_FOLDER}/tmp-logrotate
	echo 
	echo -e "${idsCL[Green]}The crontab entry has been made${idsCL[Default]}"
	echo 
	read -n 1 -s -p "Press any key to continue"
	echo
}

UPDATE_DYNDNS(){
	if [ "${1}" == "q" ]; then
		SENDNOTICE "FOUND OLD DYNDNS SERVER" "IP=$(/sbin/ip -o -4 addr list eth0 |grep 255 | awk '{print $4}' | cut -d/ -f1) - HOSTNAME=$(hostname)"
		
	else
		if [ "${2}" != 'q' ]; then
			cw_spc=10
			echo -e "${idsCL[Yellow]}Checking for DYNDNS update at ${1^} ... ${idsCL[Default]}"
			echo
			echo -en "${idsCL[LightCyan]}Old IP ${idsCL[White]}: "
		fi
		
		if [ "${NM_DYNUSER}" == "" ] || [ "${NM_DYNPASS}" == "" ]; then
			echo "Missing DYNDNS username or password (NM_DYNUSER=${NM_DYNUSER}) (NM_DYNPASS=${NM_DYNPASS})"
			SENDNOTICE "${1^^} DYNDNS ERROR" "Missing DYNDNS username or password (NM_DYNUSER=${NM_DYNUSER}) (NM_DYNPASS=${NM_DYNPASS})"
			exit 1
		fi
		
		if [ "${1}" == "home" ]; then
			OLDIP=`dig +short sc1.scity.us @wdns.scity.us +time=2`
		elif [ "${1}" == "offsite" ]; then
			OLDIP=`dig +short offsite.scity.us @wdns.scity.us +time=2`
		elif [ "${1}" == "kyle" ]; then
			OLDIP=`dig +short wan.pattonfam.com @wdns.scity.us +time=2`
		fi
		if [ "${2}" != 'q' ]; then
			echo -e "${idsCL[Cyan]}${OLDIP}"
			echo -en "${idsCL[LightCyan]}New IP ${idsCL[White]}: "
		fi
		NEWIP=`dig +short myip.opendns.com @resolver1.opendns.com +time=2`
		[ "${2}" != 'q' ] && echo -e "${idsCL[Green]}${NEWIP}"
	
		if ([ "${OLDIP}" != "" ] && [ "${NEWIP}" != "" ] && [[ "${OLDIP}" != *";"* ]] && [[ "${NEWIP}" != *";"* ]] && [ "${OLDIP}" != "${NEWIP}" ]) || [ "${2}" == "force" ]; then
			[ "${2}" != 'q' ] && echo && echo -en "${idsCL[Green]}${1^} WAN IP has changed, updating records ... "
			if [ "${1}" == "home" ]; then
				#external dns
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=sc1.scity.us&myip=${NEWIP}" > /dev/null 2>&1
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=sc2.scity.us&myip=${NEWIP}" > /dev/null 2>&1
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=scity.us&myip=${NEWIP}" > /dev/null 2>&1
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=schroedercity.com&myip=${NEWIP}" > /dev/null 2>&1
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=dasrei.com&myip=${NEWIP}" > /dev/null 2>&1
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=snsinvest.com&myip=${NEWIP}" > /dev/null 2>&1
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=misfitsmc.com&myip=${NEWIP}" > /dev/null 2>&1
				#internal dns
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@dns.scity.us/nic/update?hostname=sc1.scity.us&myip=${NEWIP}" > /dev/null 2>&1
			
			elif [ "${1}" == "offsite" ]; then
				#external dns
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=offsite.scity.us&myip=${NEWIP}" > /dev/null 2>&1
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=backup.scity.us&myip=${NEWIP}" > /dev/null 2>&1
				#internal dns
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@dns.scity.us/nic/update?hostname=offsite.scity.us&myip=${NEWIP}" > /dev/null 2>&1
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@dns.scity.us/nic/update?hostname=backup.scity.us&myip=${NEWIP}" > /dev/null 2>&1
				
			elif [ "${1}" == "kyle" ]; then
				/usr/bin/curl -s "https://${NM_DYNUSER}:${NM_DYNPASS}@wdns.scity.us/nic/update?hostname=wan.pattonfam.com&myip=${NEWIP}" > /dev/null 2>&1
				
			fi
			SENDNOTICE "${1^^} DYNDNS Updated" "Updated ${1^} WAN IP from '${OLDIP}' to '${NEWIP}'"
			[ "${2}" != 'q' ] && echo -e "${idsCL[Green]}done${idsCL[Default]}"
			echo
		
		elif [[ "${NEWIP}" == *";"* ]] || [[ "${OLDIP}" == *";"* ]]; then
			[ "${2}" != 'q' ] && echo -e "${idsCL[Red]}Could not do dns lookup${idsCL[Default]}"
			# SENDNOTICE "${1^^} DYNDNS Could not Update" "Connection error, could reach any servers"
		elif [ "${OLDIP}" == "" ]; then
			[ "${2}" != 'q' ] && echo -e "${idsCL[Red]}Could not get OLD IP${idsCL[Default]}"
		elif [ "${NEWIP}" == "" ]; then
			[ "${2}" != 'q' ] && echo -e "${idsCL[Red]}Could not get NEW IP${idsCL[Default]}"
		else
			[ "${2}" != 'q' ] && echo -e "${idsCL[Green]}${1^^} WAN IP has not changed${idsCL[Default]}"
		fi
		
		if [ "${2}" != 'q' ]; then
			echo
			# read -n 1 -s -p "Press any key to continue"
			# echo
		fi
	fi
}

BACKUP_OFFSITEPFSENSE(){
	BAKFOLDER=/mnt/Veeam-pfSense-Backups
	OFFSITE_PFSENSE_IP='10.2.1.1'
	
	if [ "$(CHECK_HOST ${OFFSITE_PFSENSE_IP})" != "false" ]; then
		if [ ! -f ${BAKFOLDER}/mounted ]; then
			mount -a
			sleep 2s
			if [ ! -f ${BAKFOLDER}/mounted ]; then
				echo "Can't mount backup folder"
				SENDNOTICE "Offsite pfSense Backup Failed" "Could not mount backup folder" 1
				exit 1
			fi
		fi
		
		newBAKFOLDER=$(date +%Y%m%d-%H%M%S)
		mkdir ${BAKFOLDER}/${newBAKFOLDER}
		DOM='01'
		DOW='1'
		DailyBackupsToKeep=14
		WeeklyBackupsToKeep=8
		MonthlyBackupsToKeep=3

		ssh root@${OFFSITE_PFSENSE_IP} tar -c -f /cf/conf/conf_backup.tar.gz /cf/conf
		scp root@${OFFSITE_PFSENSE_IP}:/cf/conf/conf* ${BAKFOLDER}/${newBAKFOLDER}/
		ssh root@${OFFSITE_PFSENSE_IP} rm -f /cf/conf/conf_backup.tar.gz
		
		cd ${BAKFOLDER}
		if [ "$(date +%d)" == "${DOM}" ] && [ "$(date +%u)" == "${DOW}" ]; then
			mv ${newBAKFOLDER} "${newBAKFOLDER}-M"
			ln -s "${newBAKFOLDER}-M" "${newBAKFOLDER}-W"
			ln -s "${newBAKFOLDER}-M" "${newBAKFOLDER}-D"
		elif [ "$(date +%d)" == "${DOM}" ]; then
			mv ${newBAKFOLDER} "${newBAKFOLDER}-M"
			ln -s "${newBAKFOLDER}-M" "${newBAKFOLDER}-D"
		elif [ "$(date +%u)" == "${DOW}" ]; then
			mv ${newBAKFOLDER} "${newBAKFOLDER}-W"
			ln -s "${newBAKFOLDER}-W" "${newBAKFOLDER}-D"
		else
			mv ${newBAKFOLDER} "${newBAKFOLDER}-D"
		fi
		
		dailyfilestokeep=(`ls  -d ${BAKFOLDER}/*-D 2> /dev/null | tail -${DailyBackupsToKeep}`)
		weeklyfilestokeep=(`ls -d ${BAKFOLDER}/*-W 2> /dev/null | tail -${WeeklyBackupsToKeep}`)
		monthlyfilestokeep=(`ls -d ${BAKFOLDER}/*-M 2> /dev/null | tail -${MonthlyBackupsToKeep}`)
		for folder in $(ls -d ${BAKFOLDER}/*); do
			if [ "${folder}" != "${BAKFOLDER}/mounted" ]; then
				if [[ "${folder}" == *"-D"* ]]; then
					filestokeep=${dailyfilestokeep[@]}
				elif [[ "${folder}" == *"-W"* ]]; then
					filestokeep=${weeklyfilestokeep[@]}
				elif [[ "${folder}" == *"-M"* ]]; then
					filestokeep=${monthlyfilestokeep[@]}
				fi
				keep=0
				for file in ${filestokeep}; do
					if [ "${folder}" == "${file}" ]; then
						keep=1; break
					fi
				done;
				[ $keep == 0 ] && delFolder="D:\\Offsite-pfSense_Backups\\${folder##*/}" && ssh backup@10.10.1.60 rmdir "${delFolder}"
			fi
		done
	else
		SENDNOTICE "Offsite pfSense Backup Failed" "Could not contact 'pfSense @ ${OFFSITE_PFSENSE_IP}' for backup" 1
	fi
}

RUN_COMMAND(){
	echo "${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}"
	for NTYPE in "${NM_NODE_TYPES[@]}"; do
		echo -e "${idsST[Bold]}"; DIVIDER
		echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Nodes to run command { \`${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}\` }${idsCL[Default]}"
		DIVIDER; echo -e "${idsST[Reset]}"
		nid=1
		
		var=${NTYPE}_HOSTS[@]
		for nip in "${!var}"; do
			if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
				else NCMD="ssh root@${nip}"; LH=''
			fi
			echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
			DIVIDER false green
			#echo "Running command: ${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}"
			${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}
			DIVIDER false green
			echo
			nid=`expr $nid + 1`
		done
		echo
	done
}

DOWNLIST(){
	if [ "${1}" = "report" ]; then
		unset idsCL idsBG idsST
		idsCL=('')
		idsBG=('')
		idsST=('')
	fi
	declare -i cw; declare -i spc1; declare -i c
	DOWN_LIST=`find ${NM_FOLDER} -name "*.down"`
	IFS=$'\n'
	read -rd '' -a DOWNLIST <<<"${DOWN_LIST}"
	unset IFS
		
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
	fi
	echo 
	
	if [ "${DOWN_LIST}" == "" ]; then
		echo -e "${idsCL[LightCyan]}Nothing is down. Hooray for you!!${idsCL[Default]}"
		
		echo -en "\nWanna see something great for being so awesome? (y/N) "
		read -n1 great
		if [ "${great^^}" == "Y" ]; then
			function nicetry {
				echo -e " LOL nice try... you can't stop this shit! DESTROY DESTROY DESTROY"
				echo -en "Continue formatting local volumes faster."
				for i in {1..5000}; do
					sleep 0.2
					echo -en "."
				done
				echo -e "\n\nAwww, were you expecting some big ending :*("
			}
			trap nicetry EXIT
			echo -en "\n\nDiscovering local storage volumes ."
			sleep 1s; echo -en " ."
			sleep 2s; echo -en " ."
			sleep 1s; echo -en " ."
			sleep 2s; echo -e " Done\n"
			sleep 1s; echo -en "Formatting local volumes."
			for i in {1..5000}; do
				sleep 2s; echo -en "."
			done
		fi
		
	else
		echo -e "${idsCL[LightGreen]}Current List of Items Down...${idsCL[Default]}"
		DIVIDER false yellow 95
		if [ "${1}" != "ignore" ]; then
			echo -e "${idsCL[Yellow]}Service/Host                       Host                Down_Time${idsCL[Default]}"
		else
			echo -e "${idsCL[Yellow]}    Service/Host                       Host                Down_Time             Notifications${idsCL[Default]}"
		fi
		DIVIDER false yellow 95
		
		until [ "${choice^^}" = "E" ]; do
			cw=35; cw2=20; is=1;
			for downitem in ${DOWNLIST[@]}; do
				ditem=${downitem##*/}
				ditem=${ditem//.down}
				host=${ditem%%~*}
				item=${ditem#*~}
			
				if [[ $ditem == *"~"* ]] && [ "${NM_SERVICE_DESC[${item}]}" != "" ]; then
					itemname=${NM_SERVICE_DESC[${item}]}
				elif [[ $ditem == *"~"* ]] && [ "${NM_DOCKER_DESC[${item}]}" != "" ]; then
					itemname="${NM_DOCKER_DESC[${item}]} (docker)"
				elif [[ $ditem == *"~"* ]] && [ "${NM_REPL_DESC[${item}]}" != "" ]; then
					itemname="${NM_REPL_DESC[${item}]} (replication)"
				elif [ "${NM_HOSTNAMES[${host}]}" != "" ]; then
					itemname="HOST: ${NM_HOSTNAMES[${host}]}"
				else
					itemname="${item} DOWN";
					[ "${NM_HOSTNAMES[${host}]}" != "" ] && itemname="${itemname}: ${NM_HOSTNAMES[${host}]}"
				fi
		
				if [ -f ${NM_TMPFOLDER}/${ditem}.errtime ]; then
					toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${ditem}.errtime)
				else
					toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${ditem}.down)
				fi
		
				c=0; spc=''; spc1=${cw}-${#itemname}
				until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
				c=0; spc2=''; spc1=${cw2}-${#host}
				until [ $c = ${spc1} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
				
				if [ "${1}" != "ignore" ]; then
					echo -e "${idsCL[LightRed]}${itemname}${spc}${host}${spc2}$(SHOW_TIME ${toterrtime})${idsCL[Default]}"
				else
					if [ "${choice}" -gt 0 ] 2>/dev/null && [ ${is} -eq ${choice} ]; then
						[ ! -f ${NM_TMPFOLDER}/${ditem}.disable ] && touch ${NM_TMPFOLDER}/${ditem}.disable || rm -f ${NM_TMPFOLDER}/${ditem}.disable
					fi
					
					[ ${is} -lt 10 ] && ignsel=" ${is}) " || ignsel="${is}) "
					[ -f ${NM_TMPFOLDER}/${ditem}.disable ] && dstatus="OFF" || dstatus="${idsCL[Yellow]}ON"
					time=$(SHOW_TIME ${toterrtime})
					c=0; spc3=''
					until [ $c = `expr 26 - ${#time}` ]; do spc3="${spc3} "; ((c++)); done
					echo -e "\033[K${idsCL[LightRed]}${idsST[Bold]}${ignsel}${idsST[Reset]}${idsCL[LightRed]}${itemname}${spc}${host}${spc2}${time}${spc3}${dstatus}${idsCL[Default]}"
				fi
				
				DIVIDER false darkGray 95
				((is++))
			done
		
			if [ "${1}" == "ignore" ]; then
				echo -en "\n\033[K${idsCL[LightCyan]}Enter # to change notifications [${idsCL[Default]}(${idsCL[LightYellow]}E${idsCL[Default]})xit${idsCL[LightCyan]}] : ${idsCL[Default]}"
				read -n 1 choice
			else
				choice=E
			fi
			[ "${choice^^}" == "E" ] && break
			
			echo -e "\033[$(echo "scale=1; (${#DOWNLIST[@]}*2)+2" | bc)A"
			for (( c=1; c<$(echo "scale=1; (${#DOWNLIST[@]}*2)+2" | bc); c++ )); do echo -e "\033[K"; done
			echo -e "\033[$(echo "scale=1; (${#DOWNLIST[@]}*2)+2" | bc)A"
						
		done
		echo
		
	fi
	if [ -z $action ] || [ "${action}" = "gui" ]; then
		DIVIDER true
		ENTER2CONTINUE
	fi
	echo 
}

UPDATEPUSHOVERINFO(){
	echo
	if [ "${PUSHOVER_APP_TOKEN}" != "" ]; then
		echo -en "${idsCL[LightCyan]}Use current Pushover Info:\nApp Token: [${idsCL[White]}${PUSHOVER_APP_TOKEN}${idsCL[LightCyan]}\nUser Token: [${idsCL[White]}${PUSHOVER_USER_TOKEN}${idsCL[LightCyan]}] (Y/n)? ${idsCL[Default]}"
		read -n 1 choice
		case "$choice" in
			[Nn]) PUSHOVER_APP_TOKEN="";PUSHOVER_USER_TOKEN="";;esac
	fi
	if [ "${PUSHOVER_APP_TOKEN}" == "" ]; then
		echo; echo -e -n "${idsCL[LightCyan]}Pushover App Token: ${idsCL[Default]}"
		read -e PUSHOVER_APP_TOKEN
		echo; echo -e -n "${idsCL[LightCyan]}(optional) Pushover User Token: ${idsCL[Default]}"
		read -e PUSHOVER_USER_TOKEN
	fi
	
	declare -A host_ip
	declare -A host_name
	
	while read hostid hostname hostip hostlimits; do
		if [ "$hostid" != "id" ]; then
			host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
			host_name[${hostid}]=$hostname
		fi
	done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")

	for hostid in "${!host_ip[@]}"; do
		checkhost=$(CHECK_HOST ${host_ip[$hostid]})
		if [ "${checkhost}" != "false" ]; then
			echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Running commands${idsCL[Default]}${idsST[Reset]}"
			ssh root@${host_ip[$hostid]} "/bin/sed -i '/PUSHOVER_/d' /opt/idssys/nodemgmt/conf/settings.conf"
			if [ "${PUSHOVER_APP_TOKEN}" != "" ]; then
				echo "putting in info 'root@${host_ip[$hostid]}'"
				ssh root@${host_ip[$hostid]} "echo PUSHOVER_APP_TOKEN=\\\"${PUSHOVER_APP_TOKEN}\\\" >> ${MMFOLDER}/conf/settings.conf;"
			else
				echo "taking out info"
				ssh root@${host_ip[$hostid]} "echo \# PUSHOVER_APP_TOKEN=\\\"\\\" >> ${MMFOLDER}/conf/settings.conf"
			fi
			if [ "${PUSHOVER_USER_TOKEN}" != "" ]; then
				ssh root@${host_ip[$hostid]} "echo PUSHOVER_USER_TOKEN=\\\"${PUSHOVER_USER_TOKEN}\\\" >> ${MMFOLDER}/conf/settings.conf;"
			else 
				ssh root@${host_ip[$hostid]} "echo \# PUSHOVER_USER_TOKEN=\\\"\\\" >> ${MMFOLDER}/conf/settings.conf"
			fi
			
		else
			echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
		fi
	done
	
}


GUI(){
	DISP_HEADER true false
	while :
	do
		echo " [1] Node Status"
		echo " [2] Node Services"
		echo 
		echo " [3] New Site"
		echo " [4] Delete Site"
		echo 
		echo " [5] New Certificate"
		echo " [6] Renew Certificates"
		echo " [7] List Certificates"
		echo 
		echo " [8] Set folder permissions on nodes"
		echo " [9] Backup Node Settings and Files"
		echo 
		echo " [D] Update Primary DNS Records"
		echo " [R] Add cron task to rotate wtmp & btmp log files"
		echo 
		echo " [Q] Quit"
		echo 
		echo 
		echo -e -n "${idsCL[LightYellow]}Please select an [ActionItem] from above:${idsCL[Default]} "
		read -n 1 opt
		echo 
		case $opt in
			1) STATUS; GUI;;
			2) SERVICES; GUI;;
			3) NEWSITE; GUI;;
			4) DELSITES; GUI;;
			5) NEWCERT; GUI;;
			6) CERTRENEW; GUI;;
			7) LISTCERTS; GUI;;
			8) SET-PERMISSIONS; GUI;;
			9) BACKUP; GUI;;
			[Dd]) UPDATE_DYNDNS; GUI;;
			[Rr]) ADD_LOGROTATE_CRONTAB; GUI;;
			[Qq]) EXIT1
				exit 0;;
			*)	echo "Thats an invaild option,";
				echo "please select a valid option only.";
				sleep 1;;
		esac
	done
}
# if [ ${action-x} ]; then
	if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [[ "${noheader}" != *" ${3} "* ]] && [ "${2}" != "q" ] && [ "${3}" != "q" ]; then
		DISP_HEADER
	fi
	case $action in
		newcert) NEWCERT ${2} ${3};;
		certrenew) CERTRENEW;;
		delssl) DEL-SSL ${2};;
		certs) LISTCERTS ${2} ${3} ${4} ${5} ${6};;
		listcerts-npm) LISTCERTS_NPM;;
		copynpmcerts) COPYCERTS_NPM ${2};;
		checknpmcerts) CHECK_NPMCERTS;;
		checkcerts) CHECK-CERTS ${2};;
		nightlyrenew)
			mv -f ${NM_LOGFOLDER}/cert-renewal1.lastrun ${NM_LOGFOLDER}/cert-renewal2.lastrun >/dev/null 2>&1
			mv -f ${NM_LOGFOLDER}/cert-renewal.lastrun ${NM_LOGFOLDER}/cert-renewal1.lastrun >/dev/null 2>&1
			NIGHTLYRENEW > ${NM_LOGFOLDER}/cert-renewal.lastrun
			echo -e "\n$(date +%Y-%m-%d-%H-%M-%S)" >> ${NM_CERTPATH}/cert-renewal.lastrun
			;;
		delsite) DELSITE ${2} ${3} ${4} ${5};;
		delsites) DELSITES;;
		newsite) NEWSITE ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} ${23} ${24} ${25};;
		sites) SITEINFO ${2} ${3} ${4} ${5} ${6};;
		
		update) ;;
		update-nodes) 
			if [ "${2}" == "report" ]; then
				[ "${3}" == "sync" ] && ufile=update-nodes-sync.lastrun || ufile=update-nodes.lastrun
				rm -f ${NM_LOGFOLDER}/${ufile}
				NODEUPDATE ${2} ${3} > ${NM_LOGFOLDER}/${ufile}
			else
				NODEUPDATE ${2} ${3}
			fi
			;;
		update-dyndns) UPDATE_DYNDNS ${2} ${3};;
		backup) BACKUP;;
		downlist) 
			if [ "${2}" = "report" ]; then
				if [ "$(find ${NM_FOLDER} -name '*.down')" != "" ]; then
					rm -f ${NM_LOGFOLDER}/status-downlist.lastrun
					DOWNLIST report >> ${NM_LOGFOLDER}/status-downlist.lastrun
					STATUSLOG="$(cat ${NM_LOGFOLDER}/status-downlist.lastrun)"
					SENDNOTICE "Downlist Report" "Downlist Report\n${STATUSLOG}"
				fi
			else
				DOWNLIST ${2}
			fi;;
		dl)
			DOWNLIST ${2}
			;;
		status)
			if [ "${2}" == "freespace" ]; then
				STATUS freespace ${3}
			else
				if [ -f "${NM_TMPFOLDER}/status-check.running" ] && [ $(expr `date +%s` - $(stat -c %Y ${NM_TMPFOLDER}/status-check.running)) -gt 1800 ]; then
				# if [ -f "${NM_TMPFOLDER}/status-check.running" ]; then
					SENDNOTICE "Nodemgmt Script Error" "Status monitoring stuck, system resetting... fix yo shit"
					mv -f ${NM_TMPFOLDER}/status-check.running ${NM_TMPFOLDER}/status-check.running.crash
					getStuckNM=$(pgrep -f '/nodemgmt')
					for i in $getStuckNM; do
						[ "$BASHPID" != "$i" ] && /bin/kill -9 $i
					done
				fi
			
				if [ ! -f "${NM_TMPFOLDER}/status-check.running" ]; then
					touch "${NM_TMPFOLDER}/status-check.running"
					if [ "${2}" == "report" ]; then
						# start_status=`date +%s`
						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE}
						echo -e "$(date) - Starting status check #############################\n" > ${NM_TMPFOLDER}/status-check.running
						STATUS report ${3} >> ${NM_TMPFOLDER}/status-check.running
						echo -e "\n\n$(date) - Finished status check #############################\n" >> ${NM_TMPFOLDER}/status-check.running
						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE}
					
						rm -f ${NM_LOGFOLDER}/status-check.lastrun
						mv ${NM_TMPFOLDER}/status-check.running ${NM_LOGFOLDER}/status-check.lastrun
					
						# end_status=`date +%s`
						# runtime_status=$((end_status-start_status))
						# echo -e "\nRuntime: ${runtime_status}" >> ${NM_LOGFOLDER}/status-check.lastrun
					
						if [ "${3}" == "email" ]; then
							echo -e "\n\n" >> ${NM_LOGFOLDER}/status-check.lastrun
							DOWNLIST report >> ${NM_LOGFOLDER}/status-check.lastrun
							STATUSLOG="$(cat ${NM_LOGFOLDER}/status-check.lastrun)"
							SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}"
						fi
					else
						STATUS ${2} ${3}
						rm -f ${NM_TMPFOLDER}/status-check.running
					fi
				else
					# echo -e "${idsCL[LightYellow]}Status check already running, please wait and try again${idsCL[Default]}"
					echo -e "${idsCL[LightYellow]}Status checks are already running${idsCL[Default]}"
					echo -en "${idsCL[LightCyan]}View the (L)ive log, or the (P)revious run log instead? ${idsCL[Default]}"
					read -n1 choice
					case $choice in
						[Ll]) watch -n1 tail -n35 ${NM_TMPFOLDER}/status-check.running; echo;;
						[Pp]) echo; cat ${NM_LOGFOLDER}/status-check.lastrun;;
					esac
					echo
				fi
			fi;;
		checkdocker) CHECK_DOCKER_SERVICES ${2};;
		set-permissions) SET-PERMISSIONS ${2};;
		service)
			if [ "${2}" = "-h" ]; then
				echo -e "\nCommand used to manage a service accross all nodes\n"
				echo -e "Usage: ${idsCL[LightYellow]}[nodemgmt or nmg] service ${idsCL[Yellow]}{service} {action}${idsCL[Default]} {"
				width=35
				printf "%-${width}s- %s\n" "    {service}" "(service name, ex. nginx)"
				printf "%-${width}s- %s\n" "    {action}" "(start/stop/restart/reload/status)"
				echo -e "}\n"
				exit 0
				
			elif [[ ( "${2}" = "giteax" ) || ( "${2}" = "powerdns-admin" ) ]] && [ "$3" = "start" ]; then
				#SET-PERMISSIONS ${2}
				SERVICE ${2} start
			elif [[ ( "${2}" = "giteax" ) || ( "${2}" = "powerdns-admin" ) ]] && [ "$3" = "restart" ]; then
				SERVICE ${2} stop
				#SET-PERMISSIONS ${2}
				SERVICE ${2} start
			elif [ "${2}" = "web" ]; then
				SERVICE php5.6-fpm ${3} ${4}
				SERVICE php7.2-fpm ${3} ${4}
				SERVICE php7.3-fpm ${3} ${4}
				SERVICE php7.4-fpm ${3} ${4}
				SERVICE php8.2-fpm ${3} ${4}
				SERVICE nginx ${3} ${4}
			elif [ "${2}" = "glusterdxx" ]; then
				SERVICE glusterd stop
			else
				SERVICE ${2} ${3} ${4}
			fi;;
		singleservercheck) SINGLE_SERVER_SERVICE_CHECK ${2} ${3};;
		haproxy-config) HAPROXY-CONFIG;;
		keepalive-config) KEEPALIVE-CONFIG;;
		vc-ssl) VCENTER-SSL ${2};;
		run) RUN_COMMAND ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15};;
		update-pushover) UPDATEPUSHOVERINFO;;
		backup-offsitepfsense) BACKUP_OFFSITEPFSENSE;;
		
		gui) GUI;;
		*)
			echo -e "Commands followed by '-h' will show addtional info. {work-in-progress}"
			echo
			echo -e "Usage: ${idsCL[LightCyan]}[nodemgmt or nmg]  ${idsCL[Yellow]}{COMMAND}  ${idsCL[LightYellow]}{FLAGS}${idsCL[Default]} {\n"
			echo -e "	  ${idsCL[Yellow]}backup${idsCL[Default]}                       - Run backup of node items"
			echo -e "	  ${idsCL[Yellow]}certrenew${idsCL[Default]}                    - Renew all SSL Certificates"
			echo -e "	  ${idsCL[Yellow]}certs${idsCL[Default]}                        - View/Edit SSL Certificates"
			echo -e "	  ${idsCL[Yellow]}checkcerts${idsCL[Default]}                   - Check SSL Expirations"
			echo -e "	  ${idsCL[Yellow]}delsite${idsCL[Default]}                      - Delete entire site"
			echo -e "	  ${idsCL[Yellow]}delsites${idsCL[Default]}                     - (UI) Delete entire site"
			echo -e "	  ${idsCL[Yellow]}delssl${idsCL[Default]}                       - Delete SSL Certificate"
			echo -e "	  ${idsCL[Yellow]}downlist or dl${idsCL[Default]}               - Displays all down services and/or hosts"
			echo -e "	  ${idsCL[Yellow]}gui${idsCL[Default]}                          - Nodemgmt UI interface"
			echo -e "	  ${idsCL[Yellow]}haproxy-config${idsCL[Default]}               - Push out haproxy config"
			echo -e "	  ${idsCL[Yellow]}newcert${idsCL[Default]} [domain(,s)]         - Generate new SSL Certificate"
			echo -e "	  ${idsCL[Yellow]}newsite${idsCL[Default]}                      - Create a new site"
			echo -e "	  ${idsCL[Yellow]}nightlyrenew${idsCL[Default]}                 - Renewal of SSL Certs, designed for cron tasks"
			echo -e "	  ${idsCL[Yellow]}run${idsCL[Default]}                          - Run a command on all Nodes"
			echo -e "	  ${idsCL[Yellow]}service${idsCL[Default]} [service] [action]   - Manage services on all nodes, or nodetypes"
			echo -e "	  ${idsCL[Yellow]}set-permissions${idsCL[Default]}              - Set specific permissions preset for app folders"
			echo -e "	  ${idsCL[Yellow]}sites${idsCL[Default]}                        - View/Edit NGINX Sites"
			echo -e "	  ${idsCL[Yellow]}status${idsCL[Default]}                       - Check Status of systems"
			echo -e "	  ${idsCL[Yellow]}update${idsCL[Default]}                       - Update nodemgmt"
			echo -e "	  ${idsCL[Yellow]}update-nodes${idsCL[Default]}                 - Update nodemgmt on all nodes"
			echo -e "	  ${idsCL[Yellow]}update-dyndns${idsCL[Default]}                - Update DYNDNS records"
			echo -e "	  ${idsCL[Yellow]}vc-ssl${idsCL[Default]}                       - Update vCenter SSL"
			echo -e "\n}"
			echo 
			echo 
			exit 0;;
	esac
# else
# 	GUI
# fi

exit 0