Update status.inc

README.md

@@ -1,4 +1,4 @@
 # NodeMgmt - Galera/NGINX Node Management
 
 
-    bash <(curl -sL https://scity.xyz/install-nodemgmt)
+    bash <(curl -sL https://go.scity.us/install-nodemgmt)

defaults.inc

@@ -1,60 +1,223 @@
 #!/usr/bin/env bash
 
-VERS='2.02072019'
+VERS='4.6.23-04302023'
-NODEHOSTS=10.5.10.51,10.5.10.52,10.5.10.53
+
-NODESERVICES=mysql,nginx,gitea,haproxy,keepalived,maxscale
+#NODETYPES=WEB,LB
-REPL_CHECKS=
+NODETYPES=MYSQL,LB,WEB,NC,WPD,LPD,MM,PW
-CERT_DAEMON='/usr/bin/certbot'
+MYSQLHOSTS=10.10.1.51,10.10.1.52
+WEBHOSTS=10.10.1.121,10.10.1.122
+LBHOSTS=10.10.10.81,10.10.10.82
+NCHOSTS=10.10.10.42
+WPDHOSTS=100.90.74.120,100.123.225.4,10.10.1.28
+LPDHOSTS=10.10.10.10,10.10.10.11
+MMHOSTS=10.10.1.24
+PRIMARYHOST=10.10.1.51
+MYSQLMANAGER=10.10.10.50
+GLHOSTS=10.10.1.41,10.10.1.42,10.10.1.121,10.10.1.122
+PWHOSTS=10.10.0.240
+
+NODESERVICES=mariadb,mysql,nginx,gitea,pdnsadmin,pdnsadmin.socket,haproxy,keepalived,maxscale,postgresql,web,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,syncthing,pdns,proxysql,radarr,sonarr,jackett,qbittorrent,snap.tautulli.tautulli,ombi,glusterd,redis-server,mosquitto
+MYSQLSERVICES_CHECK=mariadb,proxysql,keepalived
+WEBSERVICES_CHECK=nginx,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,php8.2-fpm,gitea,keepalived
+LBSERVICES_CHECK=nginx,haproxy,keepalived
+NCSERVICES_CHECK=nginx,php8.1-fpm,redis-server
+WPDSERVICES_CHECK=pdns,mariadb,pdnsadmin,pdnsadmin.socket,proxysql
+LPDSERVICES_CHECK=pdns,mariadb,pdnsadmin,pdnsadmin.socket,keepalived
+MMSERVICES_CHECK=radarr,sonarr,jackett,qbittorrent,snap.tautulli.tautulli,ombi,nginx,php8.2-fpm
+PWSERVICES_CHECK=mosquitto
+
+WEBDOCKER=phpmyadmin,phppgadmin
+#LBDOCKER=nginx-proxy_manager
+
+noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck '
+CERT_DAEMON='/snap/bin/certbot'
 FOLDER=/opt/idssys/nodemgmt
+TMPFOLDER=${FOLDER}/.tmp
 SCRIPT=${FOLDER}/nodemgmt-scripts.sh
+LOGFILE=${FOLDER}/logfile
+RENOTIFY=900
+
+[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER}
+
+
+NM_SRVCOPT=(start stop restart reload enable disable)
+
+#MYSQL_REPL_CHECK=( )
+#WEB_REPL_CHECK=(nginx www ssl lessh php pma pdnsa)
+WEB_REPL_CHECK=( web-data )
+LB_REPL_CHECK=( lb-data )
+#NC_REPL_CHECK=(apache nextcloud-php nextcloud-data)
+#NC_REPL_CHECK=( )
+
+
+
+declare -A NM_HOSTNAMES
+NM_HOSTNAMES['10.10.1.121']='Webserver-Node1'
+NM_HOSTNAMES['10.10.1.122']='Webserver-Node2'
+NM_HOSTNAMES['10.10.1.51']='MySQL-Node1'
+NM_HOSTNAMES['10.10.1.52']='MySQL-Node2'
+NM_HOSTNAMES['10.10.10.81']='LB-Node1'
+NM_HOSTNAMES['10.10.10.82']='LB-Node2'
+NM_HOSTNAMES['10.10.10.10']='DNS-Node1'
+NM_HOSTNAMES['10.10.10.11']='DNS-Node2'
+NM_HOSTNAMES['10.10.10.42']='Nextcloud-Server'
+NM_HOSTNAMES['159.203.158.51']='PowerDNS-Node1'
+NM_HOSTNAMES['146.190.1.194']='PowerDNS-Node3'
+NM_HOSTNAMES['10.10.10.50']='MySQL-Manager'
+NM_HOSTNAMES['10.10.0.240']='PowerWall'
+NM_HOSTNAMES['100.90.74.120']='WAN-DNS1 (East)'
+NM_HOSTNAMES['100.123.225.4']='WAN-DNS2 (West)'
+
+
+declare -i errtime
 
 IFS=,
-NODE_HOSTS=(${NODEHOSTS})
+NODE_TYPES=(${NODETYPES})
+MYSQL_HOSTS=(${MYSQLHOSTS})
+WEB_HOSTS=(${WEBHOSTS})
+LB_HOSTS=(${LBHOSTS})
+NC_HOSTS=(${NCHOSTS})
+WPD_HOSTS=(${WPDHOSTS})
+LPD_HOSTS=(${LPDHOSTS})
+MM_HOSTS=(${MMHOSTS})
+GL_HOSTS=(${GLHOSTS})
+PW_HOSTS=(${PWHOSTS})
+
 NODE_SERVICES=(${NODESERVICES})
+MYSQL_SERVICES_CHECK=(${MYSQLSERVICES_CHECK})
+WEB_SERVICES_CHECK=(${WEBSERVICES_CHECK})
+LB_SERVICES_CHECK=(${LBSERVICES_CHECK})
+NC_SERVICES_CHECK=(${NCSERVICES_CHECK})
+WPD_SERVICES_CHECK=(${WPDSERVICES_CHECK})
+LPD_SERVICES_CHECK=(${LPDSERVICES_CHECK})
+MM_SERVICES_CHECK=(${MMSERVICES_CHECK})
+PW_SERVICES_CHECK=(${PWSERVICES_CHECK})
 LOCAL_SERVICES=(${LOCAL_SERVICES})
+
+WEB_DOCKER=(${WEBDOCKER})
+LB_DOCKER=(${LBDOCKER})
+
 unset IFS
 
+declare -A NM_NODETYPES
+NM_NODETYPES['MYSQL']='MySQL'
+NM_NODETYPES['LB']='LoadBalance'
+NM_NODETYPES['WEB']='Webserver'
+NM_NODETYPES['NC']='Nextcloud'
+NM_NODETYPES['WPD']='WAN PowerDNS'
+NM_NODETYPES['LPD']='LAN PowerDNS'
+NM_NODETYPES['MM']='Media Manager'
+NM_NODETYPES['PW']='PowerWall'
+
 declare -A NM_SERVICES
 NM_SERVICES['mysql']='MySQL'
-NM_SERVICES['nginx']='NGINX'
+NM_SERVICES['mariadb']='MariaDB (MySQL)'
-NM_SERVICES['gogs']='Gogs'
+NM_SERVICES['postgresql']='PostgreSQL'
-NM_SERVICES['gitea']='Gitea'
+NM_SERVICES['apache2']='Apache Web Server'
-NM_SERVICES['haproxy']='HAProxy'
+NM_SERVICES['nginx']='NGINX Web Server'
-NM_SERVICES['keepalived']='Keepalived'
+NM_SERVICES['gogs']='Gogs Git Repository'
-NM_SERVICES['maxscale']='MaxScale'
+NM_SERVICES['gitea']='Gitea Git Repository'
+NM_SERVICES['haproxy']='HAProxy Load Balancer'
+NM_SERVICES['keepalived']='Keepalived Load Balancer'
+NM_SERVICES['maxscale']='MaxScale Load Balancer'
 NM_SERVICES['cmon']='CC-Controller'
 NM_SERVICES['cmon-events']='CC-Events Manager'
 NM_SERVICES['cmon-cloud']='CC-Cloud Daemon'
+NM_SERVICES['php5.6-fpm']='PHP 5.6 FPM'
+NM_SERVICES['php7.2-fpm']='PHP 7.2 FPM'
+NM_SERVICES['php7.3-fpm']='PHP 7.3 FPM'
+NM_SERVICES['php7.4-fpm']='PHP 7.4 FPM'
+NM_SERVICES['php8.0-fpm']='PHP 8.0 FPM'
+NM_SERVICES['php8.1-fpm']='PHP 8.1 FPM'
+NM_SERVICES['php8.2-fpm']='PHP 8.2 FPM'
+NM_SERVICES['syncthing']='Syncthing'
+NM_SERVICES['pdns']='PowerDNS Server'
+NM_SERVICES['pdnsadmin']='PowerDNS-Admin'
+NM_SERVICES['pdnsadmin.socket']='PowerDNS-Admin Socket'
+NM_SERVICES['proxysql']='ProxySQL Load Balancer'
+NM_SERVICES['radarr']='Radarr'
+NM_SERVICES['sonarr']='Sonarr'
+NM_SERVICES['jackett']='Jackett'
+NM_SERVICES['qbittorrent']='qBittorrent'
+NM_SERVICES['snap.tautulli.tautulli']='Tautulli'
+NM_SERVICES['ombi']='Ombi'
+NM_SERVICES['glusterd']='Gluster FileSystem'
+NM_SERVICES['redis-server']='Redis Caching Server'
+NM_SERVICES['mosquitto']='Mosquitto MQTT Broker'
+
+declare -A NM_DOCKERS
+NM_DOCKERS['phpmyadmin']='phpMyAdmin'
+NM_DOCKERS['phppgadmin']='phpPgAdmin'
+NM_DOCKERS['nginx-proxy_manager']='Nginx Proxy Manager'
+
+declare -A NM_DOCKER_COMPOSE
+NM_DOCKER_COMPOSE['phpmyadmin']='/mnt/web-Data/phpmyadmin'
+NM_DOCKER_COMPOSE['phppgadmin']='/mnt/web-Data/phppgadmin'
+NM_DOCKER_COMPOSE['nginx-proxy_manager']='/opt/nginx-proxy'
 
 declare -A NM_SRVCOPTS
+NM_SRVCOPTS['status']='Status'
 NM_SRVCOPTS['start']='Start'
 NM_SRVCOPTS['stop']='Stopp'
 NM_SRVCOPTS['restart']='Restart'
 NM_SRVCOPTS['reload']='Reload'
 NM_SRVCOPTS['enable']='Enabl'
 NM_SRVCOPTS['disable']='Disabl'
+NM_SRVCOPTS['daemon-reload']='Daemon-Reload'
 
-NM_SRVCOPT=(start stop restart reload enable disable)
 
 declare -A REPL_CHECKS
-REPL_CHECKS['nginx']='/etc/nginx'
+REPL_CHECKS['apache']='/etc/apache2'
-REPL_CHECKS['ssl']='/etc/letsencrypt'
+REPL_CHECKS['nginx']='/mnt/web-data/nginx'
-REPL_CHECKS['php']='/etc/php'
+REPL_CHECKS['nginxproxy']='/opt/nginx-proxy'
-REPL_CHECKS['www']='/var/www'
+REPL_CHECKS['lb-data']='/opt/lb-data'
-REPL_CHECKS['git']='/var/lib/gitea'
+REPL_CHECKS['ssl']='/mnt/web-data/letsencrypt'
+REPL_CHECKS['lessh']='/home/le/.ssh'
+REPL_CHECKS['php']='/mnt/web-data/php'
+REPL_CHECKS['www']='/mnt/web-data/www'
+REPL_CHECKS['web-data']='/mnt/web-data'
+REPL_CHECKS['git']='/mnt/web-data/gitea'
 REPL_CHECKS['gitssh']='/home/git'
-
+REPL_CHECKS['pma']='/mnt/web-data/phpmyadmin'
-REPL_CHECK=(nginx ssl php www git gitssh)
+REPL_CHECKS['pdnsa']='/mnt/web-data/powerdns-admin'
+REPL_CHECKS['nextcloud-data']='/mnt/Nextcloud-Data'
+REPL_CHECKS['nextcloud-php']='/etc/php'
 
 declare -A REPL_DESC
+REPL_DESC['apache']='Apache Settings'
 REPL_DESC['nginx']='NGINX Settings'
+REPL_DESC['nginxproxy']='NGINX-Proxy Settings'
+REPL_DESC['lb-data']='LB-Data (NGINX / SSL-Certs)'
 REPL_DESC['ssl']='SSL Certificates'
+REPL_DESC['lessh']='LetsEncrypt SSH Keys'
 REPL_DESC['php']='PHP Settings'
 REPL_DESC['www']='Webserver Files'
+REPL_DESC['web-data']='Web Data (NGINX/PHP)'
 REPL_DESC['git']='Gitea System'
 REPL_DESC['gitssh']='Gitea SSH Keys'
+REPL_DESC['pma']='phpMyAdmin Settings'
+REPL_DESC['pdnsa']='PowerDNS Admin'
+REPL_DESC['nextcloud-data']='Nextcloud Data'
+REPL_DESC['nextcloud-php']='Nextcloud PHP Settings'
 
+declare -A BACKUP_ITEMS
+BACKUP_ITEMS[nginx-settings]=/etc/nginx
+BACKUP_ITEMS[letsencrypt-certs]=/etc/letsencrypt
+BACKUP_ITEMS[gitea]=/var/lib/gitea
+BACKUP_ITEMS[nginx-logs]='/var/www/!NGINX-Logs'
+#BACKUP_ITEMS[webserver-files]=/var/www
 
+GET-CHECKCERT-DOMAINS(){
+	declare -A CHECKCERT_DOMAINS
+	IFS=$'\n'
+	for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
+	    HOST=${LINE%% *}
+	    PORT=${LINE#* }
+	    IFS=" "
+		CHECKCERT_DOMAINS[${HOST}]=${PORT}
+	done
+	unset IFS
+}
 
 DISP_HEADER(){
 	declare -i cw; declare -i spc1; declare -i c
@@ -62,7 +225,7 @@ DISP_HEADER(){
 		clear
 	fi
 	echo ""
-	echo -e "${idsCL[LightGreen]} NodeMgmt - Galera/NGINX Node Management${idsCL[Default]}  ${idsCL[DarkGray]}(ver-${VERS})${idsCL[Default]}"
+	echo -e "${idsCL[LightGreen]} NodeMgmt - Node Monitoring & Management${idsCL[Default]}  ${idsCL[DarkGray]}(ver-${VERS})${idsCL[Default]}"
 	DIVIDER . lightGreen
 	if [ "$2" = true ]; then
 		echo -e " Node hostname     : ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME}${idsCL[Default]}${idsST[Reset]}"
@@ -81,18 +244,25 @@ DISP_HEADER(){
 			fi
 		done
 		DIVIDER
-	fi
+		if [ $(ls -1 ${FOLDER}/*.lastrun 2>/dev/null | wc -l) != 0 ];then
-	echo ""
+			echo -e "${idsST[Bold]}Lastrun Items:${idsST[Reset]}"
-}
+			for lastrun in ${FOLDER}/*.lastrun ; do
+				IFS='/'; lastrun_item=(${lastrun}); unset IFS
+				lastrun_item=$(echo ${lastrun_item[4]} | sed "s/.lastrun//g")
+				lastrun_item=$(echo ${lastrun_item} | sed "s/-/ /g")
+				lastrun_date=$(stat -c %y ${lastrun})
+				IFS=' '; lastrun_date=(${lastrun_date}); unset IFS
+				IFS='.'; lastrun_time=(${lastrun_date[1]}); unset IFS
+				c=0; cw=18; spc=''
+				spc1=${cw}-${#lastrun_item}
+				until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+				echo -e " ${lastrun_item~}${spc}: ${lastrun_date[0]} ${lastrun_time}"
+			done
 			
-DIVIDER(){
+			DIVIDER
-	if [ -z ${2+x} ]; then clr='yellow'
+		fi
-		else clr=$2
 	fi
-	echo -e "${idsCL[${clr~}]}-----------------------------------------------------------------------${idsCL[Default]}"
-	if [ "$1" = true ]; then
 	echo ""
-	fi
 }
 
 CERT-CHECK(){
@@ -113,16 +283,15 @@ CERT-CHECK(){
 	fi
 }
 
-CHECK_HOST(){
+SENDNOTICE(){
-	if [ ! -z ${1+x} ]; then
+	[ "${PUSHOVER_APP_TOKEN}" != "" ] && PUSH_TO_MOBILE "${2}
-		ping -c 3 ${1} > /dev/null 2>&1
+
-		if [ $? -ne 0 ]; then
+$(date)" "${1}" ${3} &
-			echo false
+	
-		else
+	[ "${EMAIL_NOTICE}" != "" ] && echo -e "${2}\n\n$(date)" | mail -s "${1}" ${EMAIL_NOTICE}
-			echo true
+	
-		fi
-	else
-		echo false
-	fi
 }
 
+
+
+

inc/certs.inc

@@ -0,0 +1,438 @@
+#!/usr/bin/env bash
+
+NEWCERT(){
+	echo
+	if [ -z ${1+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}Create certificate for what name (comma seperated for mutiple) : ${idsCL[Default]}"
+		read NEW_CERT
+		echo
+	else
+		NEW_CERT=${1}
+	fi
+	if [[ ${NEW_CERT} == *","* ]]; then
+		IFS=','; NEW_CERTS=(${NEW_CERT}); unset IFS
+		MAIN_CERT=${NEW_CERTS[0]}
+	else
+		MAIN_CERT=${NEW_CERT}
+	fi
+	if [ "${NEW_CERT}" != "" ]; then
+		echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}"
+		echo ""
+		
+		# echo -en "${idsCL[LightYellow]}Stopping other Webservers... ${idsCL[Default]}"
+		# for nip in "${WEB_HOSTS[@]}"; do
+		# 	if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
+		# 		# `ssh root@${nip} service nginx stop`
+		# 		# SERVICE nginx stop ${nip}
+		# 	fi
+		# done
+		# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
+		# echo
+		
+		$CERT_DAEMON certonly --webroot -w /opt/lb-data/letsencrypt-acme-challenge -d ${NEW_CERT}
+		# $CERT_DAEMON certonly --dry-run --webroot -w /var/www/html -d ${NEW_CERT}
+		
+		chown -R root:le /opt/lb-data/letsencrypt
+		chmod -R 6775 /opt/lb-data/letsencrypt
+		
+		if [ -d /opt/lb-data/letsencrypt/live/${MAIN_CERT} ]; then
+
+			touch /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert
+			if [ -f /opt/lb-data/nginx/sites-enabled/${MAIN_CERT}.conf ]; then
+				rm -f  ${FOLDER}/cert-request.lastrun
+				daterun=`date +%Y-%m-%d-%H-%M-%S`
+				echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun
+				yes | cp -rfH ${FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun
+				DIVIDER true
+				echo -e -n "${idsCL[LightCyan]}Reload NGINX on LB Nodes (Y/n): ${idsCL[Default]}"
+				read -n 1 NGINXRELOAD
+				if [[ ${NGINXRELOAD} =~ ^[Nn]$ ]]; then
+					tmp=''
+				else
+					echo
+					echo -en "${idsCL[LightYellow]}Waiting for Cert replication across the nodes... ${idsCL[Default]}"
+					for nip in "${LB_HOSTS[@]}"; do
+						checkhost=$(CHECK_HOST ${nip})
+						if [ "${checkhost}" != "false" ]; then
+							if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
+								checked=false
+								until [ "${checked}" = "" ]; do
+									checked=`ssh root@${nip} "[ ! -f /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert ] && echo '.'"`
+								done
+							fi
+						fi
+					done
+					rm -f /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert
+					echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
+				
+					echo
+					SERVICE nginx reload
+				fi
+			fi
+			echo
+			echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}"
+		else
+			echo
+			echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}"
+		fi
+		
+		# echo
+		# echo -en "${idsCL[LightYellow]}Starting other Webservers... ${idsCL[Default]}"
+		# for nip in "${WEB_HOSTS[@]}"; do
+		# 	if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
+		# 		`ssh root@${nip} service nginx start`
+		# 		# SERVICE nginx start ${nip}
+		# 	fi
+		# done
+		# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
+		
+		echo ""
+		if [ -z $action ] || [ "${action}" = "gui" ]; then
+			DIVIDER true
+			ENTER2CONTINUE
+		fi
+	else
+		echo "You havent entered a site address."
+		sleep 1
+		NEWCERT
+		exit 0
+	fi
+}
+
+DEL-SSL(){
+	if [ -z ${1+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}Delete what SSL site address: ${idsCL[Default]}"
+		read DEL_SSL
+		echo ""
+	else
+		DEL_SSL=${1}
+	fi
+	if [ ! -z ${DEL_SSL+x} ] && [ "${DEL_SSL}" != "" ]; then
+		if [ -d /etc/letsencrypt/live/${DEL_SSL} ]; then 
+			echo -e "${idsCL[LightRed]}Deleting the SSL certificates for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}'...${idsCL[Default]}"
+			echo ""
+	
+			echo -en "${idsCL[LightRed]}Removing Files and Folders... ${idsCL[Default]}"
+			rm -rf /etc/letsencrypt/archive/${DEL_SSL}
+			rm -rf /etc/letsencrypt/live/${DEL_SSL}
+			rm -f /etc/letsencrypt/renewal/${DEL_SSL}.conf
+			echo -e "${idsCL[Green]}OK${idsCL[Default]}"
+			echo ""
+		
+			
+		else
+			echo -e "${idsCL[LightRed]}The SSL files folder for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}' could not be found.${idsCL[Default]}"
+			exit 1
+		fi
+		if [ "${timeout}" != "true" ]; then
+			echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
+		fi
+		echo
+		# SERVICE nginx reload
+		echo -e "${idsCL[LightRed]}The SSL certificate has been removed fromt be nodes.${idsCL[Default]}"
+	fi
+}
+
+CERTRENEW(){
+	echo -en "${idsCL[LightCyan]}Stopping Webserver-Node2...${idsCL[Default]}"
+	ssh root@webserver-node2.scity.us service nginx stop
+	echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}"
+	echo
+	echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}"
+	echo
+	sleep 5
+	$CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
+	# $CERT_DAEMON renew --force-renewal  --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
+    # $CERT_DAEMON --dry-run  --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
+	CONCAT_SSL
+	chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
+	chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
+	yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
+	daterun=`date +%Y-%m-%d-%H-%M-%S`
+	echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
+	DIVIDER true
+	echo -en "${idsCL[LightCyan]}Starting Webserver-Node2 Back up...${idsCL[Default]}"
+	ssh root@webserver-node2.scity.us service nginx start
+	echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}"
+	echo
+	SERVICE nginx reload 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
+	echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}"
+	echo ""
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		DIVIDER true
+		ENTER2CONTINUE
+	fi
+}
+NIGHTLYRENEW(){
+	rm -f ${FOLDER}/cert-renewal.lastrun
+	ssh root@webserver-node2.scity.us service nginx stop
+	sleep 5
+	$CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun
+	CONCAT_SSL
+	chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
+	chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
+	yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
+	daterun=`date +%Y-%m-%d-%H-%M-%S`
+	echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
+	ssh root@webserver-node2.scity.us service nginx start
+	SERVICE nginx reload web &>> ${FOLDER}/cert-renewal.lastrun
+}
+
+CONCAT_SSL(){
+	rm -f /tmp/ssllist
+	for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist
+	for certdir in $(</tmp/ssllist); do
+		rm -f ${certdir}fullcert.pem
+		cat ${certdir}privkey.pem ${certdir}fullchain.pem > ${certdir}fullcert.pem
+	done
+}
+
+LISTCERTS(){
+	declare -i cw; declare -i spc1; declare -i c
+	declare -A CHECKCERT_DOMAINS
+	IFS=$'\n'
+	for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
+	    HOST=${LINE%% *}
+	    PORT=${LINE#* }
+	    IFS=" "
+		CHECKCERT_DOMAINS[${HOST}]=${PORT}
+	done
+	unset IFS
+	if [ ! -z ${LOCAL_SERVICES+x} ]; then
+		NCMD="ssh root@${MYSQL_HOSTS[0]}"
+		#${NCMD} rm -f /tmp/ssllist
+		#${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist
+	else
+		NCMD=''
+		rm -f /tmp/ssllist
+		#for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist
+	fi
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		DIVIDER true
+	fi
+	echo 
+	echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}"
+	DIVIDER false yellow 120
+	echo -e "Subject Name               Monitored   Expires      Alternate Subject Names"
+	DIVIDER false yellow 120
+	
+	cw=30; 
+	c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
+	
+	#ssldir=$(${NCMD} find /opt/nginx-proxy/ssl/* -type l)
+	ssldir=$(${NCMD} find /opt/lb-data/letsencrypt/live/* -type d)
+	for certdir in ${ssldir[@]}; do
+		SUBJECT=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
+		SUBJECTNAMES=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV)
+		CERTEXPIRE=$(date -d "$(: | ${NCMD} openssl x509 -in ${certdir}/cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s');
+
+		SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, }
+		# SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/\n/, /g")
+		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g")
+		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g")
+		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g")
+		IFS=', '; SUBJECT_NAMES=(${SUBJECTNAMES}); unset IFS
+		DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS)));
+		
+		c=0; spc=''
+		spc1=${cw}-${#SUBJECT}
+		until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+		if [ $CERTEXPIRE -le $DUEIN ]; then
+			date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}"
+			SENDNOTICE "SSL Expiring" "${SUBJECT} expires on ${date}" 1
+		else
+			date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}"
+		fi
+		
+		if [ "${CHECKCERT_DOMAINS[${SUBJECT}]}" = "" ]; then
+			monitored='No '
+		else
+			monitored="${idsCL[Green]}Yes${idsCL[Default]}"
+		fi
+		if [ ${#SUBJECT_NAMES[@]} -lt 4 ]; then
+			echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored}     ${date}    ${SUBJECTNAMES}"
+		elif [ ${#SUBJECT_NAMES[@]} -lt 7 ]; then
+			echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored}     ${date}    ${SUBJECT_NAMES[0]}, ${SUBJECT_NAMES[1]}, ${SUBJECT_NAMES[2]}"
+			echo -e "${spc2}                      ${SUBJECT_NAMES[3]}, ${SUBJECT_NAMES[4]}, ${SUBJECT_NAMES[5]}"
+
+		fi
+		DIVIDER false darkGray 120
+	done
+	echo 
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		DIVIDER true
+		ENTER2CONTINUE
+	fi
+	echo ""
+}
+
+LISTCERTS_NPM(){
+	declare -i cw; declare -i spc1; declare -i c
+	declare -A CHECKCERT_DOMAINS
+	IFS=$'\n'
+	for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
+	    HOST=${LINE%% *}
+	    PORT=${LINE#* }
+	    IFS=" "
+		CHECKCERT_DOMAINS[${HOST}]=${PORT}
+	done
+	unset IFS
+	if [ ! -z ${LOCAL_SERVICES+x} ]; then
+		NCMD="ssh root@${MYSQL_HOSTS[0]}"
+		${NCMD} rm -f /tmp/ssllist
+		${NCMD} 'for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist
+	else
+		NCMD=''
+		rm -f /tmp/ssllist
+		for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist
+	fi
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		DIVIDER true
+	fi
+	echo 
+	echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}"
+	DIVIDER false yellow 120
+	echo -e "Subject Name               Monitored   Expires      Alternate Subject Names"
+	DIVIDER false yellow 120
+	
+	cw=30; 
+	c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
+	for certdir in $(</tmp/ssllist); do
+		SUBJECT=$(${NCMD} openssl x509 -in ${certdir}cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
+		SUBJECTNAMES=$(${NCMD} openssl x509 -in ${certdir}cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV)
+		CERTEXPIRE=$(date -d "$(: | ${NCMD} openssl x509 -in ${certdir}cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s');
+
+		SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, }
+		# SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/\n/, /g")
+		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g")
+		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g")
+		SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g")
+		IFS=', '; SUBJECT_NAMES=(${SUBJECTNAMES}); unset IFS
+		DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS)));
+		
+		c=0; spc=''
+		spc1=${cw}-${#SUBJECT}
+		until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+		if [ $DUEIN -gt $CERTEXPIRE ]; then
+			date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}"
+		else
+			date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}"
+		fi
+		
+		if [ "${CHECKCERT_DOMAINS[${SUBJECT}]}" = "" ]; then
+			monitored='No '
+		else
+			monitored="${idsCL[Green]}Yes${idsCL[Default]}"
+		fi
+		if [ ${#SUBJECT_NAMES[@]} -lt 4 ]; then
+			echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored}     ${date}    ${SUBJECTNAMES}"
+		elif [ ${#SUBJECT_NAMES[@]} -lt 7 ]; then
+			echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored}     ${date}    ${SUBJECT_NAMES[0]}, ${SUBJECT_NAMES[1]}, ${SUBJECT_NAMES[2]}"
+			echo -e "${spc2}                      ${SUBJECT_NAMES[3]}, ${SUBJECT_NAMES[4]}, ${SUBJECT_NAMES[5]}"
+
+		fi
+		DIVIDER false darkGray 120
+	done
+	echo 
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		DIVIDER true
+		ENTER2CONTINUE
+	fi
+	echo ""
+}
+COPYCERTS_NPM(){
+	ssldir='/opt/nginx-proxy/ssl'
+	
+	if [ "${1}" = "reset" ]; then
+		echo "Resetting ... Deleting all SSL symlink folders"
+		ssldirs=$(find ${ssldir} -type l)
+		for certdir in ${ssldirs[@]}; do
+			rm ${certdir}
+		done
+	else
+		baddirs=$(find $ssldir -type l ! -exec test -e {} \; -print)
+		for od in ${baddirs[@]}; do
+			echo "Removing bad symlink folder ... ${od}"
+			rm -f ${od}
+		done
+	fi
+	
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		DIVIDER true
+	fi
+	
+	ssls=$(find /opt/nginx-proxy/letsencrypt/live/* -type d)
+	for certdir in ${ssls[@]}; do
+		SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
+		if [ ! -L ${ssldir}/${SUBJECT} ]; then
+			echo "${SUBJECT} ... creating"
+			/bin/ln -s ${certdir} ${ssldir}/${SUBJECT}
+		fi
+	done
+	
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		echo
+		DIVIDER true
+		ENTER2CONTINUE
+		echo
+	fi
+}
+
+CHECK_NPMCERTS(){
+	ssls=$(find /opt/nginx-proxy/letsencrypt/live/* -type d)
+	
+	for certdir in ${ssls[@]}; do
+		SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
+		ENDDATE=$(openssl x509 -in ${certdir}/cert.pem -noout -enddate | awk -F '=' '{print $2}')
+		DAYS=$(dateutils.ddiff today "$(date --date="${ENDDATE}" '+%F')" -f '%d')
+		
+		c=0; cw=35; spc=''
+		spc1=`expr ${cw} - ${#SUBJECT}`
+		until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+		[ ${DAYS} -lt 0 ] && spc=${spc::-1}
+		
+		if [ ${DAYS} -lt 24 ]; then
+			echo "${SUBJECT}:${spc}${DAYS} days remaining"
+		fi
+		
+	done
+	
+	echo
+	
+}
+
+CHECK-CERTS(){
+	if [ "${1}" == "check" ]; then
+		rm -f ${FOLDER}/ssl-cert-check.lastrun
+		${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains > ${FOLDER}/ssl-cert-check.lastrun
+		SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)"
+		SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG"
+	else
+		# ${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains
+	    IFS=$'\n'
+	    for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`;  do
+	        HOST=${LINE%% *}
+	        PORT=${LINE#* }
+			CERTINFO=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p ${PORT} -s ${HOST} -N)
+			CERTVALID=$(echo ${CERTINFO} | awk -F' valid' '{print $1}')
+			CERTVALID=${CERTVALID: -1}
+			CERTDAYS=${CERTINFO#*days=}
+			if [ "${CERTVALID}" == "0" ] || [ "${CERTVALID}" == "E" ]; then
+				printf "%-30s: %s\n" "${HOST}" "SSL is not valid"
+				SENDNOTICE "SSL Not Valid" "The (${HOST}) SSL is not valid" 1
+				
+			elif [ ${CERTDAYS} -lt 1 ]; then
+				printf "%-30s: %s\n" "${HOST}" "SSL Expired! (${CERTDAYS} days)"
+				SENDNOTICE "SSL Expired" "The (${HOST}) SSL is expired!! (${CERTDAYS} days)" 2
+				
+			elif [ ${CERTDAYS} -lt 15 ]; then
+				printf "%-30s: %s\n" "${HOST}" "SSL is expiring in $CERTDAYS days!"
+				SENDNOTICE "SSL Expiring" "The (${HOST}) SSL is expiring in ${CERTDAYS} days" 1
+
+			else
+				printf "%-30s: %s\n" "${HOST}" "SSL is valid for ${CERTDAYS} days"
+			fi
+	    done
+		unset IFS
+	fi
+}

inc/sites.inc

@@ -0,0 +1,439 @@
+#!/usr/bin/env bash
+
+DELSITE(){
+	while [ $# -gt 0 ]; do
+	    case "$1" in
+	        -site) DEL_SITE=${2};;
+	        -ssl) DEL_SSL=${2};;
+			-list) DELSITES; exit 0;;
+	   	 	-*)
+				echo "Invalid option: '${1}' requires an argument" 1>&2
+				echo ""
+				echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
+				width=33
+				printf "%-${width}s- %s\n" "    -site {FQDN address}" "(*required)"
+				printf "%-${width}s- %s\n" "    -ssl {yes or [no]}" "Delete SSL certs as well"
+				printf "%-${width}s- %s\n" "    -list" "List sites (same as running nodemgmt delsites)"
+				echo "}"
+				exit 1;;
+	    esac
+		shift
+	done
+	if [ -z ${DEL_SITE+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}Delete what site address: ${idsCL[Default]}"
+		read DEL_SITE
+		echo ""
+	fi
+	if [[ $DEL_SSL =~ ^[Nn]$ ]]; then
+		DEL_SSL=no
+	elif [[ $DEL_SSL =~ ^[Yy]$ ]]; then
+		DEL_SSL=yes
+	elif [ -z ${DEL_SSL+x} ]; then
+		echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${DEL_SITE}' as well? [y/N]${idsCL[Default]} "
+		read DEL_SSL
+	fi
+	if [ ! -z ${DEL_SITE+x} ] && [ "${DEL_SITE}" != "" ]; then		
+		echo -e "${idsCL[LightRed]}Deleting site '${idsCL[Red]}${DEL_SITE^^}${idsCL[LightRed]}'...${idsCL[Default]}"
+		echo ""
+	
+		echo -e "${idsCL[LightRed]}[[Removing Files and Folders]]${idsCL[Default]}"
+		echo -e "${idsCL[LightRed]}-------------------------------------------${idsCL[Default]}"
+		echo
+		echo -en "${idsCL[LightCyan]}Removing files from all Nodes ... ${idsCL[Default]}"
+		ssh root@10.10.1.120 rm -f /etc/nginx/sites-enabled/${DEL_SITE}* >/dev/null 2>&1
+		ssh root@10.10.10.80 rm -f /etc/nginx/sites-enabled/${DEL_SITE}* >/dev/null 2>&1
+		if [ "${DEL_SSL}" == "yes" ]; then
+			ssh root@10.10.10.80 rm -rf /etc/letsencrypt/archive/${DEL_SITE} >/dev/null 2>&1
+			ssh root@10.10.10.80 rm -rf /etc/letsencrypt/live/${DEL_SITE} >/dev/null 2>&1
+			ssh root@10.10.10.80 rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf >/dev/null 2>&1
+		fi
+		echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+		echo
+		echo -en "${idsCL[LightCyan]}Reloading NGINX ... ${idsCL[Default]}"
+		ssh root@10.10.10.80 nodemgmt service nginx reload >/dev/null 2>&1
+		ssh root@10.10.1.120 nodemgmt service nginx reload >/dev/null 2>&1
+		echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+		
+		# nid=1
+		# for nip in "${WEB_HOSTS[@]}"; do
+		# 	if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
+		# 		nip='localhost '
+		# 		NCMD=''
+		# 	else
+		# 		NCMD="ssh root@${nip}"
+		# 	fi
+		# 	echo -en "Removing from Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
+		# 	if [ "${NCMD}" != "" ]; then
+		# 		checkhost=$(CHECK_HOST ${nip})
+		# 	fi
+		# 	if [ "${checkhost}" != "false" ]; then
+		# 		#if [ -f /etc/nginx/sites-available/${DEL_SITE} ]; then
+		# 			${NCMD} rm -f /etc/nginx/sites-available/${DEL_SITE}*
+		# 			${NCMD} rm -f /etc/nginx/sites-enabled/${DEL_SITE}*
+		# 			#fi
+		# 		#if [ -d /var/www/${DEL_SITE} ]; then
+		# 			${NCMD} rm -rf /var/www/${DEL_SITE}
+		# 			#fi
+		# 		if [ "${DEL_SSL}" = "yes" ]; then
+		# 			${NCMD} rm -rf /etc/letsencrypt/archive/${DEL_SITE}
+		# 			${NCMD} rm -rf /etc/letsencrypt/live/${DEL_SITE}
+		# 			${NCMD} rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf
+		# 		fi
+		# 		echo -e "${idsCL[Green]}OK${idsCL[Default]}"
+		# 	else
+		# 		echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
+		# 	fi
+		# 	nid=`expr $nid + 1`
+		# done
+		# echo ""
+		# SERVICE nginx reload ns
+		echo -e "${idsCL[LightRed]}Site has been deleted.${idsCL[Default]}"
+	else
+		echo "Missing arguments"
+		echo ""
+		echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
+		width=33
+		printf "%-${width}s- %s\n" "    -site {FQDN address}" "Site to delete"
+		printf "%-${width}s- %s\n" "    -ssl {yes or [no]}" "Delete SSL certs as well"
+		printf "%-${width}s- %s\n" "    -list" "List sites (same as running nodemgmt delsites)"
+		echo "}"
+		exit 1
+	fi
+}
+
+DELSITES(){
+	echo
+	echo -e "${idsCL[Red]}Select a site to delete...${idsCL[Default]}"
+	DIVIDER true
+	sid=1
+	filels="( $(ssh root@${WEB_HOSTS[0]} ls '/etc/nginx/sites-available/*') )"
+	# IFS='\n'
+	for siteconf in $filels; do
+	# for siteconf in /etc/nginx/sites-available/* ; do
+		# [ -e "$siteconf" ] || continue
+		if [ ${siteconf:0:1} == '/' ]; then
+			IFS='/'; site_conf=(${siteconf}); unset IFS
+			SITES[${sid}]=${site_conf[4]}
+			sid=`expr $sid + 1`
+		fi
+	done
+	for s in "${!SITES[@]}"; do
+		echo " [${s}] ${SITES[${s}]}"
+	done
+	echo ""
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		echo " [B] Back"
+	fi
+	echo " [Q] Quit"
+	echo ""
+	echo -e -n "${idsCL[LightYellow]}Please select a site from above from above:${idsCL[Default]} "
+	read selsite
+	echo ""
+	if [ -z ${SITES[$selsite]} ] && [ "${selsite}" != "Q" ] && [ "${selsite}" != "q" ] && [ "${selsite}" != "B" ] && [ "${selsite}" != "b" ]; then
+		echo "Thats an invaild option,"
+		echo "please select a valid option only."
+		sleep 1
+		DELSITES
+		exit 0
+	elif [ "${selsite}" = "Q" ] || [ "${selsite}" = "q" ]; then
+		exit 0
+	elif [ "${selsite}" = "B" ] || [ "${selsite}" = "b" ]; then
+		GUI
+	else
+		while :
+		do
+			echo -e -n "${idsCL[LightRed]}Are you sure you want to delete '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}'? [y/N]${idsCL[Default]} "
+			read response
+			echo ""
+			if [[ $response =~ ^[Yy]$ ]]; then
+				echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}', if they exist? [y/N]${idsCL[Default]} "
+				read sslresponse
+				DELSITE -site ${SITES[${selsite}]} -ssl ${sslresponse}
+				echo ""
+				DIVIDER
+				ENTER2CONTINUE
+				break
+			else
+				break
+			fi			
+		done
+		DELSITES
+		exit 0
+	fi
+	if [ -z $action ] || [ "${action}" = "gui" ]; then
+		ENTER2CONTINUE
+	fi
+}
+
+NEWSITE(){
+	echo
+	while [ $# -gt 0 ]; do
+		case "$1" in
+			-site) NEW_SITE=${2};;
+			-type) SITE_TYPE=${2};;
+			-ssl) CREATE_SSL=${2};;
+			-proxy_scheme) PROXYSCHEME=${2};;
+			-proxy_host) PROXYHOST=${2};;
+			-proxy_port) PROXYPORT=${2};;
+			-websocket) WEBSOCKET=${2};;
+			-hsts) HSTS=${2};;
+			-exploits) EXPLOITS=${2};;
+			-secure) SECURE=${2};;
+			-h | -help | --help)
+				echo ""
+				echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {"
+				width=33
+				printf "%-${width}s- %s\n" "    -site {FQDN address(,es)}" "(new site and aliases, comma separated)"
+				printf "%-${width}s- %s\n" "    -ssl {yes or no}" "(defaults to yes)"
+				printf "%-${width}s- %s\n" "    -type {'local' or 'proxy'}" "(defaults to local)"
+				printf "%-${width}s- %s\n" "    -proxy_port {host port}" "(proxy backend host)"
+				printf "%-${width}s- %s\n" "    -proxy_host {IP or FQDN}" "(proxy backend port)"
+				printf "%-${width}s- %s\n" "    -proxy_scheme {http or https}" "(proxy backend scheme)"
+				printf "%-${width}s- %s\n" "    -websocket {yes or no}" "(websocket support)"
+				printf "%-${width}s- %s\n" "    -hsts {yes or no}" "(hsts support)"
+				printf "%-${width}s- %s\n" "    -exploits {yes or no}" "(block exploits)"
+				printf "%-${width}s- %s\n" "    -secure {yes or no}" "(secure access [nginx/.htpasswd])"
+				echo "}"
+				exit 1;;
+		esac
+		shift
+	done
+		
+	#if [ -z ${SITE_TYPE+x} ]; then SITE_TYPE=local; fi
+	#if [ -z ${CREATE_SSL+x} ]; then CREATE_SSL=true; fi
+	if [ -z ${NEW_SITE+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}New site domain name (comma seperated for multiple) : ${idsCL[Default]}"
+		read NEW_SITE
+		showdivide=yes
+	fi
+	
+	if [ -z ${CREATE_SSL+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}Create SSL for site? [Y/n] ${idsCL[Default]}"
+		read CREATE_SSL
+		showdivide=yes
+		if [[ $CREATE_SSL =~ ^[Yy]$ ]] || [ "${CREATE_SSL}" = "" ]; then
+			CREATE_SSL=yes
+			# echo -e -n "${idsCL[LightCyan]}Add additonal domain names to the SSL cert (comma seperated)? : ${idsCL[Default]}"
+			# read ssladd
+		else
+			CREATE_SSL=no
+		fi
+	fi
+
+	if [ -z ${SITE_TYPE+x} ]; then
+		echo -e -n "${idsCL[LightCyan]}Site type (local/{proxy}): ${idsCL[Default]}"
+		read SITE_TYPE
+		showdivide=yes
+		if [ "${SITE_TYPE}" = "" ]; then
+			SITE_TYPE=proxy
+		fi
+	fi
+	if [ "${SITE_TYPE}" = "proxy" ]; then
+		if [ -z ${PROXYHOST+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}What is the proxy backend address (IP or FQDN): ${idsCL[Default]}"
+			read PROXYHOST
+			showdivide=yes
+		fi
+		if [ -z ${PROXYPORT+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}What is the proxy backend port (tcp port): ${idsCL[Default]}"
+			read PROXYPORT
+			showdivide=yes
+		fi
+		if [ -z ${PROXYSCHEME+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}What is the proxy backend scheme (http/https): ${idsCL[Default]}"
+			read PROXYSCHEME
+			showdivide=yes
+		fi
+		if [ -z ${WEBSOCKET+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}Enable Websocket Support (y/N): ${idsCL[Default]}"
+			read WEBSOCKET
+			showdivide=yes
+			if [[ ${WEBSOCKET} =~ ^[Nn]$ ]] || [ "${WEBSOCKET}" = "" ]; then
+				WEBSOCKET=no
+			elif [[ ${WEBSOCKET} =~ ^[Yy]$ ]]; then
+				WEBSOCKET=yes
+			else
+				WEBSOCKET=no
+			fi
+		fi
+		if [ -z ${HSTS+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}Enable HSTS Support (Y/n): ${idsCL[Default]}"
+			read HSTS
+			showdivide=yes
+			[[ ${HSTS} =~ ^[Yy]$ ]] || [ "${HSTS}" = "" ] &&HSTS=yes || HSTS=no
+		fi
+		if [ -z ${EXPLOITS+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}Block exploits (y/N): ${idsCL[Default]}"
+			read EXPLOITS
+			showdivide=yes
+			if [[ ${EXPLOITS} =~ ^[Nn]$ ]] || [ "${EXPLOITS}" = "" ]; then
+				EXPLOITS=no
+			elif [[ ${EXPLOITS} =~ ^[Yy]$ ]]; then
+				EXPLOITS=yes
+			else
+				EXPLOITS=no
+			fi
+		fi
+		if [ -z ${SECURE+x} ]; then
+			echo -e -n "${idsCL[LightCyan]}Secure site with passwords from [ nginx/.htpasswd ](y/N): ${idsCL[Default]}"
+			read SECURE
+			showdivide=yes
+			if [[ ${SECURE} =~ ^[Nn]$ ]] || [ "${SECURE}" = "" ]; then
+				SECURE=no
+			elif [[ ${SECURE} =~ ^[Yy]$ ]]; then
+				SECURE=yes
+			else
+				SECURE=no
+			fi
+		fi
+	fi
+	[ "${showdivide}" == "yes" ] && DIVIDER
+	echo
+	width=14
+	printf "%-${width}s: %s\n" "New site" "${NEW_SITE}"
+	printf "%-${width}s: %s\n" "Create SSL" "${CREATE_SSL}"
+	printf "%-${width}s: %s\n" "Site type" "${SITE_TYPE}"
+	if [ "${SITE_TYPE}" = "proxy" ]; then
+		printf "%-${width}s: %s\n" "Proxy host" "${PROXYHOST}"
+		printf "%-${width}s: %s\n" "Proxy port" "${PROXYPORT}"
+		printf "%-${width}s: %s\n" "Proxy scheme" "${PROXYSCHEME}"
+		printf "%-${width}s: %s\n" "Websocket Support" "${WEBSOCKET}"
+		printf "%-${width}s: %s\n" "HSTS Support" "${HSTS}"
+		printf "%-${width}s: %s\n" "Block Exploits" "${EXPLOITS}"
+		printf "%-${width}s: %s\n" "Secure Access" "${SECURE}"
+	fi
+	echo -e -n "${idsCL[LightRed]}Is this information correct? [Y/n]${idsCL[Default]} "
+	read -n 1 response
+	echo
+	if [[ $response =~ ^[Yy]$ ]] || [ "${response}" = "" ]; then 
+		if [[ ${NEW_SITE} == *","* ]]; then
+			IFS=','; NEW_SITES=(${NEW_SITE}); unset IFS
+			MAIN_SITE=${NEW_SITES[0]}
+			NGINX_SERVERNAME=${NEW_SITE//[,]/ }
+		else
+			MAIN_SITE=${NEW_SITE}
+			NGINX_SERVERNAME=${NEW_SITE}
+				
+		fi
+		if [ "${SITE_TYPE}" = "proxy" ]; then
+			if [ ! -z ${PROXYSCHEME+x} ] && [ ! -z ${PROXYHOST+x} ] && [ ! -z ${PROXYPORT+x} ]; then GO=true; fi
+			else GO=true
+		fi
+		if [ "${GO}" = "true" ]; then
+			echo -e "${idsCL[LightGreen]}Setting up new site for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}"
+			echo ""
+			
+			[ "${WEBSOCKET}" == "yes" ] && WEBSOCKET="include conf.d\/include\/websocket-support.conf;" || WEBSOCKET=""
+			[ "${HSTS}" == "yes" ] && HSTS="include conf.d\/include\/hsts-support.conf;" || HSTS=""
+			[ "${EXPLOITS}" == "yes" ] && EXPLOITS="include conf.d\/include\/block-exploits.conf;" || EXPLOITS=""
+			[ "${SECURE}" == "yes" ] && SECURE="include conf.d\/include\/secure-access.conf;" || SECURE=""
+			
+			
+			######################################### LOCAL
+			if [ "${SITE_TYPE}" = "local" ]; then
+				echo -e "server {
+	listen 80;" > /etc/nginx/sites-available/${MAIN_SITE}
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "	listen 443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "
+	server_name ${NGINX_SERVERNAME};
+	
+	set \$base /var/www/${MAIN_SITE};
+	root \$base/public_html;
+
+	access_log  /var/log/nginx/${MAIN_SITE}-access.log;
+	error_log  /var/log/nginx/${MAIN_SITE}-error.log warn;" >> /etc/nginx/sites-available/${MAIN_SITE}
+
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "
+	ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem;
+	ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem;
+	include conf.d/include/ssl-ciphers.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "
+	index index.php;
+
+	location / {
+		try_files \$uri \$uri/ /index.php?\$query_string;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "		include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "	}
+
+	location ~ \.php\$ {
+		fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
+		include conf.d/include/php_fastcgi.conf;
+	}
+
+	include conf.d/include/general.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				if [ "${CREATE_SSL}" = "yes" ]; then
+					echo -e "	include conf.d/include/letsencrypt-acme-challenge.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
+				fi
+				echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE}
+				
+
+				sudo -u www-data mkdir -p /var/www/${MAIN_SITE}/{public_html,nginx_logs}
+				# echo -en "${idsCL[LightYellow]}Waiting for folder replication across the webserver nodes... ${idsCL[Default]}"
+				# for nip in "${WEB_HOSTS[@]}"; do
+				# 	checkhost=$(CHECK_HOST ${nip})
+				# 	if [ "${checkhost}" != "false" ]; then
+				# 		if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
+				# 			checked=false
+				# 			until [ "${checked}" = "" ]; do
+				# 				checked=`ssh root@${nip} "[ ! -d /var/www/${MAIN_SITE} ] && echo does not exist"`
+				# 			done
+				# 		fi
+				# 	fi
+				# done
+				# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
+				#
+				# echo -en "${idsCL[LightYellow]}Setting folder permissions... ${idsCL[Default]}"
+				# SET-PERMISSIONS ${MAIN_SITE}
+				# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
+				
+				
+							
+			######################################### PROXY
+			else
+				
+				cp ${FOLDER}/templates/nginx.proxy.site /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				sed -i "s/<<SERVER_NAME>>/${NGINX_SERVERNAME}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				sed -i "s/<<MAIN_SITE>>/${MAIN_SITE}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				sed -i "s/<<PROXY_IP>>/${PROXYHOST}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				sed -i "s/<<PROXY_PORT>>/${PROXYPORT}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				sed -i "s/<<PROXY_SCHEME>>/${PROXYSCHEME}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				sed -i "s/<<WEBSOCKET>>/${WEBSOCKET}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				sed -i "s/<<HSTS>>/${HSTS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				sed -i "s/<<EXPLOITS>>/${EXPLOITS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				sed -i "s/<<SECURE>>/${SECURE}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+			fi
+			
+			if [ "${CREATE_SSL}" = "yes" ]; then
+				[ "${ssladd}" != "" ] && NEWCERT ${NEW_SITE},${ssladd} || NEWCERT ${NEW_SITE}
+				if [ "${SITE_TYPE}" == "proxy" ]; then
+					sed -i "s/#ssl_certificate/ssl_certificate/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
+				fi
+			fi
+			
+			rm -f  ${FOLDER}/new-site.lastrun
+			daterun=`date +%Y-%m-%d-%H-%M-%S`
+			echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun
+			# yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun
+			# yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun
+			# daterun=`date +%Y-%m-%d-%H-%M-%S`
+			# echo -e "${daterun}" >> /etc/nginx/new-site.lastrun
+			DIVIDER true
+			echo ""
+			echo -e "${idsCL[LightGreen]}The new site for '${idsCL[LightGreen]}${NEW_SITE}${idsCL[Default]}' has been created.${idsCL[Default]}"
+			echo ""
+		else
+			echo "Missing proxy arguments"
+			Exit 1
+		fi
+
+	else
+		${SCRIPT} newsite
+		exit 0
+	fi
+}

inc/status.inc

@@ -0,0 +1,684 @@
+#!/usr/bin/env bash
+
+STATUS(){
+	start=`date +%s`
+	ST_ACTION=${1}
+	if [ "${ST_ACTION}" = "report" ]; then
+		unset idsCL idsBG idsST
+		idsCL=('')
+		idsBG=('')
+		idsST=('')
+	elif [ "${ST_ACTION}" = "sync" ]; then
+		ST_ACTION=repl
+	fi
+	declare -i cw; declare -i spc1; declare -i c
+	
+	########################
+	if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
+		if [ "${ST_ACTION}" != "check" ]; then
+			echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}"
+		fi
+		for NTYPE in "${NODE_TYPES[@]}"; do
+			PH=${NTYPE}_HOSTS[0]
+			if [ ! -f ${FOLDER}/${!PH}.down ]; then
+				PH_CMD="ssh root@${!PH}"
+				var=${NTYPE}_REPL_CHECK[@]
+				if [ ! -z ${!var+x} ]; then
+					for rcheck in "${!var}"; do
+						# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
+						${PH_CMD} rm -f ${FOLDER}/test.repl
+						daterun=`date +%Y-%m-%d-%H-%M-%S`
+						${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" &
+					done
+				fi
+			fi
+		done
+		# replstart=`date +%s`
+		if [ "${ST_ACTION}" != "check" ]; then
+			echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
+			echo
+		fi
+	fi
+
+	########################
+	if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then
+		if [ ! -z ${LOCAL_SERVICES+x} ]; then
+			if [ "${ST_ACTION}" != "check" ]; then
+				lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
+				uptime=`uptime -p`
+				echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
+				DIVIDER false green
+			fi
+			for srvc in "${LOCAL_SERVICES[@]}"; do
+				if [ "${ST_ACTION}" != "check" ]; then
+					if [ "${ST_ACTION}" != "report" ]; then
+						c=0; cw=30; spc=''
+						spc1=${cw}-${#NM_SERVICES[${srvc}]}
+						until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+					else
+						spc=' '
+					fi
+					echo -en " ${NM_SERVICES[${srvc}]}${spc}: "
+				fi
+				
+				if [ "$(systemctl is-active ${srvc})" != "active" ]; then
+					if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
+						if [ "${ST_ACTION}" != "check" ]; then
+							echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
+						fi
+						SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1
+						touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
+						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
+					else
+						errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
+						if [ $errtime -gt ${RENOTIFY} ]; then
+							if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
+								mv ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
+							fi
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+							fi
+							touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
+						fi
+					fi
+				else
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+					fi
+					if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
+						if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
+						else
+							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
+						fi
+						SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
+						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.*
+						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE}
+						
+					fi
+				fi
+			done
+			
+			if [ "${ST_ACTION}" != "check" ]; then echo; fi
+		fi
+		########################
+		
+		for NTYPE in "${NODE_TYPES[@]}"; do
+			nid=1
+			
+			dockers=${NTYPE}_DOCKER[@]
+			hosts=${NTYPE}_HOSTS[@]
+			
+			var=${NTYPE}_HOSTS[@]
+			
+			if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" == "dockers" ]; then
+				GOFORCHECK=false;
+			else
+				GOFORCHECK=true;
+			fi
+				
+			if [ ${GOFORCHECK} = true ]; then
+			
+				if [ "${ST_ACTION}" != "check" ]; then
+					echo -e "${idsST[Bold]}"; DIVIDER
+					if [ "${ST_ACTION}" = "dockers" ]; then
+						echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
+					else
+						echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
+					fi
+					DIVIDER; echo -e "${idsST[Reset]}"
+				fi
+				for nip in "${!var}"; do
+					[ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}"
+					
+					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
+						else NCMD="ssh root@${nip}"; LH=''
+					fi
+					if [ "${NCMD}" != "" ]; then
+						checkhost=$(CHECK_HOST ${nip})
+					fi
+					if [ "${checkhost}" != "false" ]; then
+						if [ "${ST_ACTION}" != "check" ]; then
+							#echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}"
+							if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then
+								uptime=`${NCMD} uptime -p`
+								echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+							else
+								echo -e "${idsCL[Default]}"
+							fi
+							
+							DIVIDER false green
+						fi
+						if [ -f ${FOLDER}/${nip}.down ]; then
+							if [ -f ${FOLDER}/${nip}.errtime ]; then
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+							else
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+							fi
+							rm -f ${FOLDER}/${nip}.*
+							SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
+						fi
+						
+						if [ "${ST_ACTION}" != "dockers" ]; then
+							srvcs=${NTYPE}_SERVICES_CHECK[@];
+							srvcstotest="$(join_by " " ${!srvcs})"
+							srvctst=(`${NCMD} systemctl is-active ${srvcstotest}`)
+							sr=0
+							for srvc in "${!srvcs}"; do
+								[ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false || NOGOCHK=true
+								if [ ${NOGOCHK} == true ]; then
+									if [ "${ST_ACTION}" != "check" ]; then
+										if [ "${ST_ACTION}" != "report" ]; then
+											c=0; cw=30; spc=''; spc1=${cw}-${#NM_SERVICES[${srvc}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+										else
+											spc=' '
+										fi
+										echo -en " ${NM_SERVICES[${srvc}]}$spc: "
+									fi
+									if [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" == "mysql" ]; then
+										[ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false
+									elif [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
+										[ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false
+									elif [ "${srvctst[$sr]}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then
+										mysqlgo=true
+									fi
+									# echo "mysqlgo=${mysqlgo}"
+									if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
+										if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
+											[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
+											SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
+											touch ${FOLDER}/${nip}-${srvc}.down
+											echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
+										else
+											errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+											if [ $errtime -gt ${RENOTIFY} ]; then
+												[ ! -f ${FOLDER}/${nip}-${srvc}.errtime ] && mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
+												toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+												touch ${FOLDER}/${nip}-${srvc}.down
+												# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
+											fi
+											[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+										fi
+									else
+										[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+										if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
+											[ -f ${FOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+											SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
+											echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
+
+											rm -f ${FOLDER}/${nip}-${srvc}.*
+
+										fi
+										# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
+										if [ "${srvc}" == "gitea" ]; then
+											rm -f ${FOLDER}/*-${srvc}.*
+										fi
+									fi
+									[ "${mysqlgo}" = "true" ] && unset mysqlgo
+								fi
+								sr=`expr $sr + 1`
+							done
+							
+							
+							
+							
+							
+							# for srvc in "${!var2}"; do
+							# 	NOGOCHK=true;
+							# 	[ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false;
+							# 	if [ ${NOGOCHK} = true ]; then
+							# 		if [ "${ST_ACTION}" != "check" ]; then
+							# 			if [ "${ST_ACTION}" != "report" ]; then
+							# 				c=0; cw=30; spc=''
+							# 				spc1=${cw}-${#NM_SERVICES[${srvc}]}
+							# 				until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+							# 			else
+							# 				spc=' '
+							# 			fi
+							# 			echo -en " ${NM_SERVICES[${srvc}]}$spc: "
+							# 		fi
+							# 		srvctst=$(${NCMD} systemctl is-active ${srvc})
+							# 		if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then
+							# 			mysqlgo=false;
+							# 			[ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true;
+							# 		elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
+							# 			mysqlgo=false;
+							# 			[ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true;
+							# 		elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then
+							# 			mysqlgo=true
+							# 		fi
+							# 		# echo "mysqlgo=${mysqlgo}"
+							# 		if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
+							# 			if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
+							# 					if [ "${ST_ACTION}" != "check" ]; then
+							# 						echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
+							# 					fi
+							# 					SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
+							# 					touch ${FOLDER}/${nip}-${srvc}.down
+							# 					echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
+							#
+							# 			else
+							# 				errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+							# 				if [ $errtime -gt ${RENOTIFY} ]; then
+							# 					if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+							# 						mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
+							# 					fi
+							# 					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+							# 					touch ${FOLDER}/${nip}-${srvc}.down
+							# 					if [ "${srvc}" = "mysql" ]; then
+							# 						${NCMD} systemctl restart ${srvc}
+							# 					fi
+							# 				fi
+							# 				if [ "${ST_ACTION}" != "check" ]; then
+							# 					echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+							# 				fi
+							# 			fi
+							# 		else
+							# 			if [ "${ST_ACTION}" != "check" ]; then
+							# 				echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+							# 			fi
+							# 			if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
+							# 				if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
+							# 					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
+							# 				else
+							# 					toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
+							# 				fi
+							# 				SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
+							# 				echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
+							#
+							# 				rm -f ${FOLDER}/${nip}-${srvc}.down
+							# 				rm -f ${FOLDER}/${nip}-${srvc}.errtime
+							#
+							# 			fi
+							# 			# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
+							# 			if [ "${srvc}" == "gitea" ]; then
+							# 				rm -f ${FOLDER}/*-${srvc}.down
+							# 				rm -f ${FOLDER}/*-${srvc}.errtime
+							# 			fi
+							# 		fi
+							# 		[ "${mysqlgo}" = "true" ] && unset mysqlgo
+							# 	fi
+							# done
+						fi
+						
+						if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then
+							if [ "${ST_ACTION}" != "check" ]; then
+								echo
+								echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
+								echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}"
+							fi
+						
+							for docker in "${!dockers}"; do
+								if [ "${ST_ACTION}" != "check" ]; then
+									if [ "${ST_ACTION}" != "report" ]; then
+										c=0; cw=30; spc=''
+										spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
+										until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+									else
+										spc=' '
+									fi
+									echo -en " ${NM_DOCKERS[${docker}]}$spc: "
+								fi
+								
+								if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
+									if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
+										${NCMD} docker rm ${docker} >/dev/null 2>&1
+									fi
+									${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
+									sleep 10s
+									if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
+										if [ "${ST_ACTION}" != "check" ]; then
+											echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
+										fi
+										SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed"
+										rm -f ${FOLDER}/${nip}-${docker}.down
+										rm -f ${FOLDER}/${nip}-${docker}.errtime
+										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE}
+
+									else
+										if [ "${ST_ACTION}" != "check" ]; then
+											echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
+										
+											if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
+												SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1
+												touch ${FOLDER}/${nip}-${docker}.down
+												echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE}
+											else
+												errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down)
+												if [ $errtime -gt ${RENOTIFY} ]; then
+													if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then
+														mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime
+													fi
+													toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime)
+													if [ "${ST_ACTION}" != "check" ]; then
+														echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
+													fi
+													touch ${FOLDER}/${nip}-${docker}.down
+												fi
+											fi
+										
+										fi
+									fi
+								else
+									if [ "${ST_ACTION}" != "check" ]; then
+										echo -e "${idsCL[Green]}Running${idsCL[Default]}"
+									fi
+									rm -f ${FOLDER}/${nip}-${docker}.down
+									rm -f ${FOLDER}/${nip}-${docker}.errtime
+								fi
+							done
+						fi
+					
+					else
+						if [ ! -f ${FOLDER}/${nip}.down ]; then
+							touch ${FOLDER}/${nip}.down
+							if [ ! -f ${FOLDER}/${nip}.errtime ]; then
+								touch ${FOLDER}/${nip}.errtime
+							fi
+							if [ "${ST_ACTION}" != "check" ]; then
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+								echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
+							fi
+							SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1
+							echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE}
+						else
+							if [ "${ST_ACTION}" != "check" ]; then
+								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
+								echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+							fi
+							errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
+							# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then
+							# 	echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}
+							# fi
+						fi
+
+					fi
+					if [ "${ST_ACTION}" != "check" ]; then echo; fi
+					nid=`expr $nid + 1`
+				done
+
+			fi #GOFORCHECK
+		
+		done
+		
+	fi
+	########################
+	## REPLICATION CHECK
+	########################
+	
+	if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
+		if [ "${ST_ACTION}" != "check" ]; then
+			echo -e "${idsST[Bold]}"; DIVIDER
+			echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
+			DIVIDER; echo -e "${idsST[Reset]}"
+			echo -en " ${idsCL[LightCyan]}Starting processes to collect/monitor replication status : "
+		fi
+		for NTYPE in "${NODE_TYPES[@]}"; do
+			PH=${NTYPE}_HOSTS[0]
+			PH_CMD="ssh root@${!PH}"
+			var=${NTYPE}_REPL_CHECK[@]
+			if [ ! -z ${!var+x} ]; then
+				var=${NTYPE}_HOSTS[@]
+				for nip in "${!var}"; do
+					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
+						if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
+							var2=${NTYPE}_REPL_CHECK[@]
+							for rcheck in "${!var2}"; do
+								REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
+							done
+						fi
+					fi
+				done
+			fi
+		done
+		[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"	
+		
+		for NTYPE in "${NODE_TYPES[@]}"; do
+			PH=${NTYPE}_HOSTS[0]
+			PH_CMD="ssh root@${!PH}"
+			var=${NTYPE}_REPL_CHECK[@]
+			if [ ! -z ${!var+x} ]; then
+				var=${NTYPE}_HOSTS[@]
+				for nip in "${!var}"; do
+					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
+						if [ "${ST_ACTION}" != "check" ]; then
+							echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
+							DIVIDER false green
+						fi
+						if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
+							var2=${NTYPE}_REPL_CHECK[@]
+							for rcheck in "${!var2}"; do
+								if [ "${ST_ACTION}" != "check" ]; then
+									if [ "${ST_ACTION}" != "report" ]; then
+										c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+									else
+										spc=' '
+									fi
+									echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
+
+								fi
+								
+								checked=""
+								until [ "${checked}" != "" ]; do
+									if [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
+										checked=good
+									elif [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
+										checked=timeout
+									fi
+									rm -f ${TMPFOLDER}/repl.${rcheck}.${nip}.*
+								done
+								if [ "${checked}" == "timeout" ]; then
+									[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
+									if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+										touch ${FOLDER}/${nip}-${rcheck}.down
+										SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
+										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
+									fi
+								else
+									[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
+									if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
+										rm -f ${FOLDER}/${nip}-${rcheck}.down
+										SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
+										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
+									fi
+								fi				
+							done
+						else
+							if [ "${ST_ACTION}" != "check" ]; then
+								[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}"
+								[ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}"
+							fi
+						fi
+						if [ "${ST_ACTION}" != "check" ]; then echo; fi
+					fi
+				done
+			fi
+		done
+		##########################
+		# REMOVE REPL CHECK FILES
+		##########################
+		if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
+			if [ "${ST_ACTION}" != "check" ]; then
+				echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
+			fi
+			#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
+			#LBHCMD="ssh root@${LB_HOSTS[0]}"
+			#WHCMD="ssh root@${WEB_HOSTS[0]}"
+			for NTYPE in "${NODE_TYPES[@]}"; do
+				PH=${NTYPE}_HOSTS[0]
+				if [ ! -f ${FOLDER}/${!PH}.down ]; then
+					PH_CMD="ssh root@${!PH}"
+					var=${NTYPE}_REPL_CHECK[@]
+					if [ ! -z ${!var+x} ]; then
+						for rcheck in "${!var}"; do
+							${PH_CMD} rm -f ${FOLDER}/test.repl
+							daterun=`date +%Y-%m-%d-%H-%M-%S`
+							if [ "${PH_CMD}" = "" ]; then
+								rm -f ${FOLDER}/test.repl &
+								rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
+							else
+								${PH_CMD} rm -f ${FOLDER}/test.repl &
+								${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
+							fi
+						done
+					fi
+				fi
+			done
+			rm -Rf ${FOLDER}/test.repl
+			if [ "${ST_ACTION}" != "check" ]; then
+				echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
+				echo
+			fi
+		fi	
+	fi
+	
+	########################
+	## FREE SPACE CHECK
+	########################
+	#if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then
+	if [ "${ST_ACTION}" = "freespace" ]; then
+		if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then
+			for NTYPE in "${NODE_TYPES[@]}"; do
+				if [ "${ST_ACTION}" != "check" ]; then
+					echo -e "${idsST[Bold]}"; DIVIDER
+					echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Free Space Scan${idsCL[Default]}"
+					DIVIDER; echo -e "${idsST[Reset]}"
+				fi
+				nid=1
+				var=${NTYPE}_HOSTS[@]
+				for nip in "${!var}"; do
+					if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
+						else NCMD="ssh root@${nip}"; LH=''
+					fi
+					if [ "${ST_ACTION}" != "check" ]; then
+						echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
+						DIVIDER false green
+					fi
+					
+					[ "${ST_ACTION}" != "check" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
+					declare -A partitions
+					DRIVEINFO=$(ssh root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
+					DRIVEINFO=(${DRIVEINFO})
+					echo -en "\e[1A";
+					echo -e "\e[0K\r"
+					
+					NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
+					for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
+						ii=$((${i}*3))
+						pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
+						pname=${pname#*vg-}
+						if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then
+							freespace=${DRIVEINFO[$((${ii}+2))]//M/}
+							if [ "${ST_ACTION}" != "check" ]; then
+								c=0; cw=20; spc=''
+								spc1=${cw}-${#pname}
+								until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+								echo -en " ${pname}$spc: "
+							fi
+							if [[ ${freespace} -gt 1024 ]]; then
+								fsgb=$(bc <<< "scale=2; ${freespace}/1024")
+								fsdsp="${fsgb} GB"
+							else
+								fsdsp="${freespace} MB"
+							fi
+							if [ "${freespace}" -le "1024" ]; then
+								fs_status='error'
+								fs_status_color='Red'
+								SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free" 1
+							elif [ "${freespace}" -le "5120" ]; then
+								fs_status='warn'
+								fs_status_color='Yellow'
+								SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free"
+							else
+								fs_status=''
+								fs_status_color='Green'
+							fi
+							[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
+						fi
+					done
+					
+					# for partition in "${partitions3[@]}"; do
+					# 	if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then
+					# 		pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
+					# 		pname=${pname#*vg-}
+					# 		if [ "${ST_ACTION}" != "check" ]; then
+					# 			c=0; cw=20; spc=''
+					# 			spc1=${cw}-${#pname}
+					# 			until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
+					# 			echo -en " ${pname}$spc: "
+					# 		fi
+					#
+					# 		# [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition}
+					# 		# freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1
+					#
+					# 		if [ "${freespace}" -le "1000" ]; then
+					# 			fs_status='error'
+					# 			fs_status_color='Red'
+					# 			SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" 1
+					#
+					# 		elif [ "${freespace}" -le "5000" ]; then
+					# 			fs_status='warn'
+					# 			fs_status_color='Yellow'
+					# 			SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free"
+					#
+					# 		else
+					# 			fs_status=''
+					# 			fs_status_color='Green'
+					#
+					# 		fi
+					# 		if [ "${ST_ACTION}" != "check" ]; then
+					# 			if [[ ${freespace} -gt 1000 ]]; then
+					# 				fsgb=$(bc <<< "scale=2; ${freespace}/1000")
+					# 				fsdsp="${fsgb} GB"
+					# 			else
+					# 				fsdsp="${freespace} MB"
+					# 			fi
+					# 			echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
+					#
+					# 		fi
+					# 	fi
+					# done
+
+					nid=`expr $nid + 1`
+					echo
+				done
+			done
+		fi
+	fi
+	########################
+	########################
+	
+	if [ "${ST_ACTION}" != "check" ]; then
+		echo ""
+		if [ -z $action ] || [ "${action}" = "gui" ]; then
+			DIVIDER true
+			ENTER2CONTINUE
+		fi
+	fi
+	end=`date +%s`
+	runtime=$((end-start))
+	echo "runtime: ${runtime}"
+}
+
+
+REPLCHECK(){
+	rcheck=${1}
+	nip=${2}
+	PH_CMD=${3}
+	if [ "${4}" != "check" ]; then
+		timeout=`date --date='1 minutes' +%s`
+	else
+		timeout=`date --date='2 minutes' +%s`
+	fi
+	checked=false
+	until [ "${checked}" == "" ]; do
+		if [ "${PH_CMD}" == "" ]; then
+			ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
+		else
+			ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
+		fi
+		[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
+	done
+	[ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good
+}

run.sh

@@ -1,45 +1,68 @@
 #!/usr/bin/env bash
 # NodeMgmt Loader
 
-action=$1
+if [ -f /opt/idssys/settings/nodemgmt.conf ]; then
+	mv /opt/idssys/settings/nodemgmt.conf /opt/idssys/nodemgmt/settings.conf
+fi
 
 source /opt/idssys/defaults/colors.inc
 source /opt/idssys/defaults/default.inc
-if [ ! -f /opt/idssys/settings/nodemgmt.conf ]; then
+source /opt/idssys/nodemgmt/settings.conf
-	source /opt/idssys/nodemgmt/settings.conf
-else
-	source /opt/idssys/settings/nodemgmt.conf
-fi
 source /opt/idssys/nodemgmt/defaults.inc
 
-if [ "$1" != "service" ] && [ "$1" != "status-check" ] && [ "$1" != "nightlyrenew" ] && [ "$1" != "backup" ]; then
+if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
-	if [ "${1}" = "gui" ]; then
+	# if [ "${1}" = "gui" ]; then
-		DISP_HEADER
+	# 	DISP_HEADER
-	fi
+	# fi
-	if [ "${1}" != "gui" ]; then
+	
+	# if [ "${1}" != "guiX" ]; then
+	if curl -m 15 -s --head  --request GET https://git.schroedercity.com | grep "HTTP/2 200" > /dev/null; then 
+		if [ "${2}" != "q" ]; then
 			echo -en "${idsCL[LightCyan]}Checking for updates...${idsCL[Default]}"
 			echo ""
+		fi
 		cd /opt/idssys/defaults
 		if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then
+			if [ "${2}" != "q" ]; then
+				echo -en "\e[1A";
+				echo -e "\e[0K\r ${idsCL[LightCyan]}Installing updates...${idsCL[Default]}"
+			fi
 			git fetch origin master >/dev/null 2>&1
 			git reset --hard origin/master >/dev/null 2>&1
 			git reflog expire --expire=now --all >/dev/null 2>&1
 			git repack -ad >/dev/null 2>&1
 			git prune >/dev/null 2>&1
 			git pull >/dev/null 2>&1
+			git submodule update --remote >/dev/null 2>&1
 		fi
-		cd /opt/idssys/nodemgmt
+		cd ${FOLDER}
-		if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then 
+		# if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then
-			git fetch origin master >/dev/null 2>&1
+		if [ "`git log --pretty=%H ...refs/heads/dev^ | head -n 1`" != "`git ls-remote origin -h refs/heads/dev |cut -f1`" ]; then
-			git reset --hard origin/master >/dev/null 2>&1
+			if [ "${2}" != "q" ]; then
+				echo -en "\e[1A";
+				echo -e "\e[0K\r ${idsCL[LightCyan]}Installing updates...${idsCL[Default]}"
+			fi
+			# git fetch origin master >/dev/null 2>&1
+			# git reset --hard origin/master >/dev/null 2>&1
+			git fetch origin dev >/dev/null 2>&1
+			git reset --hard origin/dev >/dev/null 2>&1
 			git reflog expire --expire=now --all >/dev/null 2>&1
 			git repack -ad >/dev/null 2>&1
 			git prune >/dev/null 2>&1
 			git pull >/dev/null 2>&1
-		fi
+			git submodule update --remote >/dev/null 2>&1
+			/bin/chmod +x /opt/idssys/nodemgmt/nodemgmt-scripts.sh 2>&1
+			if [ "${2}" != "q" ]; then
 				echo -en "\e[1A";
 				echo -e "\e[0K\r ${idsCL[Green]}Updates Completed${idsCL[Default]}"
 			fi			
+		elif [ "${2}" != "q" ]; then
+			echo -en "\e[1A";
+			echo -e "\e[0K\r ${idsCL[LightCyan]}No Updates Available${idsCL[Default]}"
+		fi
+	else
+		echo -e "${idsCL[Red]}Could not connect to 'git.schroedercity.com' for updates${idsCL[Default]}"
+	fi
 fi
-/opt/idssys/nodemgmt/nodemgmt-scripts.sh ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}
+${SCRIPT} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} ${23} ${24} ${25}
 exit 0

runcerts.old

@@ -1,226 +0,0 @@
-#!/bin/bash
-# My Cert Processes for LetsEncrypt
-
-Color_Off='\033[0m'
-Blue='\033[0;34m'
-BBlue='\033[1;34m'
-Green='\033[0;32m'
-LGreen='\033[1;32m'
-BOrange='\033[0;33m'
-DGray='\033[1;30m'
-Red='\033[10;31m'
-LRed='\033[1;31m'
-
-DAEMON='/usr/bin/certbot'
-
-redirect_artica-ssl_files () {
-  MYDIR="/etc/nginx/certificates"
-  DIRS=`ls -l $MYDIR | egrep '^d' | awk '{print $9}'`
-  for DIR in $DIRS; do
-    if [ $DIR != 'new' ] && [ $DIR != 'default' ]
-    then
-      echo -e "${LGreen}Checking cetificate redirections for \"${LRed}${DIR}${Green}\"...${Color_Off}"
-
-#rm -f /etc/nginx/certificates/${DIR}/CAkey.pem
-#rm -f /etc/nginx/certificates/${DIR}/certificate.pem
-#rm -f /etc/nginx/certificates/${DIR}/private_key.key
-#mv /etc/nginx/certificates/${DIR}/CAkey.pem.bak /etc/nginx/certificates/${DIR}/CAkey.pem
-#mv /etc/nginx/certificates/${DIR}/certificate.pem.bak /etc/nginx/certificates/${DIR}/certificate.pem
-#mv /etc/nginx/certificates/${DIR}/private_key.key.bak /etc/nginx/certificates/${DIR}/private_key.key
-
-      if [ ! -f /etc/nginx/certificates/${DIR}/CAkey.pem.bak ]
-      then
-        certs_array=('certs1=(CAkey.pem private_key.key certificate.pem)' 'certs2=(chain.pem privkey.pem fullchain.pem)')
-        for elt in "${certs_array[@]}";do eval $elt;done
-        for ((i=0;i<=2;i++));
-        do
-          mv /etc/nginx/certificates/${DIR}/${certs1[${i}]} /etc/nginx/certificates/${DIR}/${certs1[${i}]}.bak
-          ln -s /etc/letsencrypt/live/${DIR}/${certs2[${i}]} /etc/nginx/certificates/${DIR}/${certs1[${i}]}
-          echo -e "${Green}Certificate redirected for \"${DGray}.../${DIR}/${BOrange}${certs1[${i}]}${Green}\"${Color_Off}"
-        done
-      else
-        echo -e "${Green}No changes needed to be made at this time.${Color_Off}"
-      fi
-      echo
-    fi
-  done
-}
-
-case "$1" in
-
- start)
-	sleep 15
-	echo -e "${BBlue}HAProxy Starting...${Color_Off}"
-	service haproxy start
-	ssh root@10.5.10.52 service haproxy start
-	ssh root@10.5.10.53 service haproxy start
-	echo -e "${BBlue}NGINX Reload...${Color_Off}"
-	service nginx reload
-	ssh root@10.5.10.52 service nginx reload
-	ssh root@10.5.10.53 service nginx reload
-
-	exit 0
- ;;
-
- stop)
-	echo -e "${BBlue}HAProxy Stopping...${Color_Off}"
-	service haproxy stop
-	ssh root@10.5.10.52 service haproxy stop
-	ssh root@10.5.10.53 service haproxy stop
-	exit 0;;
-
- gogs-start)
-	echo -e "${BBlue}Gogs Starting...${Color_Off}"
-	service gogs start
-	ssh root@10.5.10.52 service gogs start
-	ssh root@10.5.10.53 service gogs start
-	exit 0;;
-
- gogs-stop)
-	echo -e "${BBlue}Gogs Stopping...${Color_Off}"
-	service gogs stop
-	ssh root@10.5.10.52 service gogs stop
-	ssh root@10.5.10.53 service gogs stop
-	exit 0;;
-
- gogs-restart)
-	echo -e "${BBlue}Gogs Restarting...${Color_Off}"
-	service gogs restart
-	ssh root@10.5.10.52 service gogs restart
-	ssh root@10.5.10.53 service gogs restart
-	exit 0;;
-
- nginx-start)
-	echo -e "${BBlue}NGINX Starting...${Color_Off}"
-	service nginx start
-	ssh root@10.5.10.52 service nginx start
-	ssh root@10.5.10.53 service nginx start
-	exit 0;;
-
- nginx-stop)
-	echo -e "${BBlue}NGINX Stopping...${Color_Off}"
-	service nginx stop
-	ssh root@10.5.10.52 service nginx stop
-	ssh root@10.5.10.53 service nginx stop
-	exit 0;;
-
- nginx-restart)
-	echo -e "${BBlue}NGINX Restarting...${Color_Off}"
-	service nginx restart
-	ssh root@10.5.10.52 service nginx restart
-	ssh root@10.5.10.53 service nginx restart
-	exit 0;;
-
- reload)
-	service nginx reload
-	ssh root@10.5.10.52 service nginx reload
-	ssh root@10.5.10.53 service nginx reload
-	exit 0
- ;;
-
- new)
-	# "$0" stop
-	echo -e "${Green}Requesting Certificate for '${BBlue}$2${Green}'...${Color_Off}"
-	# $DAEMON certonly --standalone -d $2
-	$DAEMON certonly --webroot -w /var/www/html -d $2
-	# chown -R root:letsencrypt /etc/letsencrypt
-	# chmod -R 6775 /etc/letsencrypt
-	# "$0" start
-	# redirect_artica-ssl_files
-	exit 0
- ;;
-
- new-mass)
-	echo -e "${Green}Requesting Certificate for '${BBlue}$2${Green}'...${Color_Off}"
-	# $DAEMON certonly --standalone -d $2
-	$DAEMON certonly --webroot -w /var/www/html -d $2
-	# chown -R root:letsencrypt /etc/letsencrypt
-	# chmod -R 6775 /etc/letsencrypt
-	# redirect_artica-ssl_files
-	exit 0
- ;;
-
- renew)
-	"$0" stop
-	echo -e "${Green}Renewing Certificates...${Color_Off}"
-	# $DAEMON renew
-	$DAEMON renew certonly --webroot -w /var/www/html -d $2
-	chown -R root:letsencrypt /etc/letsencrypt
-	chmod -R 6775 /etc/letsencrypt
-	"$0" start
-	# redirect_artica-ssl_files
-	exit 0
- ;;
-
- nightlyrenew)
-	rm -f /opt/runcerts/cert-renewal-run-$(hostname)
-	"$0" stop >> /opt/runcerts/cert-renewal-run-$(hostname)
-	# $DAEMON renew >> /opt/runcerts/cert-renewal-run-$(hostname)
-	$DAEMON renew certonly --webroot -w /var/www/html -d $2 >> /opt/runcerts/cert-renewal-run-$(hostname)
-	chown -R root:letsencrypt /etc/letsencrypt
-	chmod -R 6775 /etc/letsencrypt
-	"$0" start >> /opt/runcerts/cert-renewal-run-$(hostname)
-	exit 0
- ;;
-
-
- redirect-ssl)
-	redirect_artica-ssl_files
-	service nginx restart
-
-	read -n 1 -t 10 -p "Are you sure you wish to reboot (y/N)?" redirect_choice
-	case "$redirect_choice" in 
-	[Yy]) if [ "$EUID" -ne 0 ]
-	      then
-		service artica-webservices restart
-	      fi
-	      exit 0;;
-	esac
-	exit 0
- ;;
-
- *)
-	echo
-	echo
-	echo "Usage: $0 {new|new-mass|renew|redirect-ssl|stop|start}"
-	echo
-	echo "'new' - Creates a new certificate."
-	echo "---------------------------------------------------------------------------------------"
-	echo "examples:"
-	echo "  single-domain    = 'runcerts new www.example.com'"
-	echo "  multiple-domains = 'runcerts new www.example.com,dev.example.com,...'"
-	echo
-	echo "'new-mass' - Creates a new certificate, but doesnt go through the"
-	echo "             process of stopping and restarting nginx each time"
-	echo "---------------------------------------------------------------------------------------"
-	echo "examples (same as 'new'):"
-	echo "  single-domain    = 'runcerts new-mass www.example.com'"
-	echo "  multiple-domains = 'runcerts new-mass www.example.com,dev.example.com,...'"
-	echo
-	echo "'renew' - Renews all certificates. This command could be run routinly by adding this"
-	echo "                                   line into crontab: \"0 4 * * 0 /usr/local/bin/runcerts renew\""
-	echo "---------------------------------------------------------------------------------------"
-	echo "examples:"
-	echo "  'runcerts renew'"
-	echo
-	echo "'redirect-ssl' - Redirects the configured Artica certificates to use the LetsEncrypt certificates"
-	echo "---------------------------------------------------------------------------------------"
-	echo "examples:"
-	echo "  'runcerts redirect-ssl'"
-	echo
-	echo "'stop' - Stops any process that interfere with creating a standalone server"
-	echo "---------------------------------------------------------------------------------------"
-	echo "examples:"
-	echo "  'runcerts stop'"
-	echo
-	echo "'start' - Starts any process that had interfered with creating a standalone server"
-	echo "---------------------------------------------------------------------------------------"
-	echo "examples:"
-	echo "  'runcerts start'"
-	echo
-
-	exit 1
- ;;
-
-esac
-exit 0

ssl-cert-check/ssl-cert-check

@@ -9,16 +9,226 @@
 # Author: Matty < matty91 at gmail dot com >
 #
 # Current Version: 3.30
-
+#
+# Revision History:
+#
+# Version 3.31
+#  - Fixed the test for the -servername flag -- Kitson Consulting.
+#
+# Version 3.30
+#  - Use highest returncode for Nagios output -- Marcel Pennewiss
+#  - Set RETCODE to 3 (unknown) if a certificate file does not exist -- Marcel Pennewiss
+#  - Add a "-d" option to specify a directory or file mask pattern -- Marcel Pennewiss
+#  - Add a "-N" option to create summarized Nagios output -- Marcel Pennewiss
+#  - Cleaned up many formatting -- Marcel Pennewiss
+#
+# Versione 3.29a
+#  - Added option to specify email sender address
+#
+# Version 3.29
+#  - Add the openssl -servername flag if it shows up in help.
+#
+# Version 3.28
+#  - Added a DEBUG option to assist with debugging folks who use the script
+#
+# Version 3.27
+#  - Allow white spaces to exist in the certificate file list
+#  - Add an additional check to pick up bad / non-existent certificates
+#  - Add a check to look for the existence of a mail program. Error out if it's not present.
+#  - Enable the TLS -servername extension by default - Juergen Knaack & Johan Denoyer
+#
+# Version 3.26
+#  - Allow the certificate type (PEM, DER, NET) to be passed on the command line
+#
+# Version 3.25
+#   - Check for "no route to host" errors -- Dan Doyle
+#   - Set RETCODE to 3 (unknown) if a connection error occurs -- Dan Doyle
+#   - Documentation fixes
+#
+# Version 3.24
+#   - Utilize the -clcerts option to limit the results to client certificates - Eitan Katznelson
+#
+# Version 3.23
+#   - Fixed typo in date2julian routine -- Ken Cook
+#
+# Version 3.22
+#   - Change the validation option to "-V"
+#   - Add a "-v" option to specify a specific protocol version (ssl2, ssl3 or tls)
+#
+# Version 3.21
+#   - Adjust e-mail checking to avoid exiting if notifications aren't enabled -- Nick Anderson
+#   - Added the number of days until expiration to the Nagios output -- Nick Anderson
+#
+# Version 3.20
+#   - Fixed a bug in certificate length checking -- Tim Nowaczyk
+#
+# Version 3.19
+#   - Added check to verify the certificate retrieved is valid
+#
+# Version 3.18
+#   - Add support for connecting to FTP servers -- Paul A Sand
+#
+# Version 3.17
+#   - Add support for connecting to imap servers -- Joerg Pareigis
+#
+# Version 3.16
+#   - Add support for connecting to the mail sbmission port -- Luis E. Munoz
+#
+# Version 3.15
+#   - Adjusted the file checking logic to use the correct certificate -- Maciej Szudejko
+#   - Add sbin to the default search paths for OpenBSD compatibility -- Alex Popov
+#   - Use cut instead of substring processing to ensure compatibility -- Alex Popov
+#
+# Version 3.14
+#   - Fixed the Common Name parser to handle DN's where the CN is not the last item
+#     eg. EmailAddr -- Jason Brothers
+#   - Added the ability to grab the serial number -- Jason Brothers
+#   - Added the "-b" option to print results without a header -- Jason Brothers
+#   - Added the "-v" option for certificate validation -- Jason Brothers
+#
+# Version 3.13
+#   - Updated the subject line to include the hostname as well as
+#     the common name embedded in the X509 certificate (if it's
+#     available) -- idea proposed by Mike Burns
+#
+#  Version 3.12
+#   - Updated the license to allow redistribution and modification
+#
+#  Version 3.11
+#   - Added ability to comment out lines in files passed
+#     to the "-f" option -- Brett Stauner
+#   - Fixed comment next to file processing logic
+#
+#  Version 3.10
+#   - Fixed POP3 port -- Simon Matter
+#
+#  Version 3.9
+#    - Switched binary location logic to use which utility
+#
+#  Version 3.8
+#    - Fixed display on 80 column displays
+#    - Cleaned up the formatting
+#
+#  Version 3.7
+#    - Fixed bug in NAGIOS tests -- Ben Allen
+#
+#  Version 3.6
+#    - Added support for certificates stored in PKCS#12 databases -- Ken Gallo
+#    - Cleaned up comments
+#    - Adjusted variables to be more consistent
+#
+#  Version 3.5
+#    - Added support for NAGIOS -- Quanah Gibson-Mount
+#    - Added additional checks for mail -- Quanah Gibson-Mount
+#    - Convert tabs to spaces -- Quanah Gibson-Mount
+#    - Cleaned up usage() routine
+#    - Added additional checks for openssl
+#
+#  Version 3.4
+#   - Added a missing "{" to line 364 -- Ken Gallo
+#   - Move mktemp to the start of the main body to avoid errors
+#   - Adjusted default binary paths to make sure the script just works
+#     w/ Solaris, BSD and Linux hosts
+#
+#  Version 3.3
+#   - Added common name from X.509 certificate file to E-mail body / header -- Doug Curtis
+#   - Fixed several documentation errors
+#   - Use mktemp to create temporary files
+#   - Convert printf, sed and awk to variables
+#   - Check for printf, sed, awk and mktemp binaries
+#   - Add additional logic to make sure mktemp returned a valid temporary file
+#
+#  Version 3.2
+#   - Added option to list certificates in the file passed to "-f".
+#
+#  Version 3.1
+#   - Added handling for starttls for smtp -- Marco Amrein
+#   - Added handling for starttls for pop3 (without s) -- Marco Amrein
+#   - Removed extra spacing at end of script
+#
+#  Version 3.0
+#   - Added "-i" option to print certificate issuer
+#   - Removed $0 from Subject line of outbound e-mails
+#   - Fixed some typographical errors
+#   - Removed redundant "-b" option
+#
+#  Version 2.0
+#    - Fixed an issue with e-mails formatting incorrectly
+#    - Added additional space to host column -- Darren-Perot Spruell
+#    - Replaced GNU date dependency with CHRIS F. A. JOHNSON's
+#      date2julian shell function. This routine can be found on
+#      page 170 of Chris's book "Shell Scripting Recipes: A
+#      Problem-Solution Approach," ISBN #1590594711. Julian function
+#      was created based on a post to comp.unix.shell by Tapani Tarvainen.
+#    - Cleaned up function descriptions
+#    - Removed several lines of redundant code
+#    - Adjusted the help message
+#
+#   Version 1.1
+#    - Added "-c" flag to report expiration status of a PEM encoded
+#      certificate -- Hampus Lundqvist
+#    - Updated the prints messages to display the reason a connection
+#      failed (connection refused, connection timeout, bad cert, etc)
+#    - Updated the GNU date checking routines
+#    - Added checks for each binary required
+#    - Added checks for connection timeouts
+#    - Added checks for GNU date
+#    - Added a "-h" option
+#    - Cleaned up the documentation
+#
+#  Version 1.0
+#      Initial Release
+#
+# Last Updated: 12-12-2016
+#
+# Purpose:
+#  ssl-cert-check checks to see if a digital certificate in X.509 format
+#  has expired. ssl-cert-check can be run in interactive and batch mode,
+#  and provides facilities to alarm if a certificate is about to expire.
+#
+# License:
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#  GNU General Public License for more details.
+#
+# Requirements:
+#   Requires openssl
+#
+# Installation:
+#   Copy the shell script to a suitable location
+#
+# Tested platforms:
+#  -- Solaris 9 using /bin/bash
+#  -- Solaris 10 using /bin/bash
+#  -- OS X 10.4.2 using /bin/bash
+#  -- OpenBSD using /bin/sh
+#  -- FreeBSD using /bin/sh
+#  -- Centos Linux 3, 4, 5 & 6 using /bin/bash
+#  -- Redhat Enterprise Linux 3, 4, 5 & 6 using /bin/bash
+#  -- Gentoo using /bin/bash
+#
+# Usage:
+#  Refer to the usage() sub-routine, or invoke ssl-cert-check
+#  with the "-h" option.
+#
+# Examples:
+#   Please refer to the following site for documentation and examples:
+#   http://prefetch.net/articles/checkcertificate.html
 
 PATH=/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/ssl/bin:/usr/sfw/bin
 export PATH
 
 # Who to page when an expired certificate is detected (cmdline: -e)
-ADMIN="david@sc"
+ADMIN="root"
 
 # Email sender address for alarm notifications
-SENDER="iDS-CertChecker@scity.us"
+SENDER="postmaster@localhost.localdomain"
 
 # Number of days in the warning threshhold (cmdline: -x)
 WARNDAYS=30

ssl-cert-check/ssldomains

@@ -0,0 +1,21 @@
+cloud.schroedercity.com 443
+dns.scity.us 443
+dyndns.scity.us 443
+git.schroedercity.com 443
+go.scity.us 443
+mail.scity.us 443
+media.scity.us 443
+mysql.scity.us 443
+pma.scity.us 443
+plex.scity.us 32400
+plex.schroedercity.com 443
+pwd.schroedercity.com 443
+status.scity.us 443
+statusint.scity.us 443
+stdiscosrv.scity.us 443
+unifistats.scity.us 443
+vcenter.scity.us 443
+vpn.scity.us 4433
+wdns.scity.us 443
+www.schroedercity.com 443
+www.scity.us 443

ssldomains

@@ -1,3 +0,0 @@
-cloud.schroedercity.com 443
-git.schroedercity.com 443
-www.schroedercity.com 443

templates/haproxy.config

@@ -0,0 +1,146 @@
+global
+	pidfile /var/run/haproxy.pid
+    daemon
+    user haproxy
+    group haproxy
+    stats socket /var/run/haproxy.socket    user haproxy    group haproxy    mode 600 level admin
+    node haproxy_%NIP%
+    description haproxy    server
+
+    #* Performance Tuning
+    maxconn 8192
+    spread-checks 3
+    quiet
+defaults
+    #log global
+    mode tcp
+    option  dontlognull
+    option tcp-smart-accept
+    option tcp-smart-connect
+    #option dontlog-normal
+    retries 3
+    option redispatch
+    maxconn 8192
+    timeout check 3500ms
+    timeout queue 3500ms
+    timeout connect 3500ms
+    timeout client 10800s
+    timeout server 10800s
+
+    userlist STATSUSERS
+    group admin users admin
+    user admin insecure-password Dc$@1910
+    user stats insecure-password Dc$@1910
+
+listen admin_page
+    bind *:9600
+    mode http
+    stats enable
+    stats refresh 60s
+    stats uri /
+    acl AuthOkay_ReadOnly http_auth(STATSUSERS)
+    acl AuthOkay_Admin http_auth_group(STATSUSERS) admin
+    stats http-request auth realm admin_page unless AuthOkay_ReadOnly
+    # stats admin if AuthOkay_Admin
+
+listen MySQL_3307_rw
+    bind *:3307
+    mode tcp
+    timeout client 10800s
+    timeout server 10800s
+    tcp-check expect string is\ running
+    balance leastconn
+    option tcp-check
+    # option allbackups
+    default-server port 9200 inter 2s downinter 5s rise 3 fall 2 slowstart 60s    maxconn 64 maxqueue 128 weight 100
+    server 10.5.10.51 10.5.10.51:3306 check
+    server 10.5.10.52 10.5.10.52:3306 check backup
+    server 10.5.10.53 10.5.10.53:3306 check backup
+
+
+listen MySQL_3308_ro
+    bind *:3308
+    mode tcp
+    timeout client 10800s
+    timeout server 10800s
+    tcp-check expect string is\ running
+    balance leastconn
+    option tcp-check
+    # option allbackups
+    default-server port 9200 inter 2s downinter 5s rise 3 fall 2 slowstart 60s    maxconn 64 maxqueue 128 weight 100
+    server 10.5.10.51 10.5.10.51:3306 check
+    server 10.5.10.52 10.5.10.52:3306 check
+    server 10.5.10.53 10.5.10.53:3306 check backup
+
+listen NGINX_http
+    bind *:80
+    mode tcp
+    timeout client 10800s
+    timeout server 10800s
+    # tcp-check expect string is\ running
+    balance leastconn
+    option tcp-check
+    # option allbackups
+    default-server port 9200 inter 2s downinter 5s rise 3 fall 2 slowstart 60s    maxconn 64 maxqueue 128 weight 100
+    cookie SERVERUSED insert indirect nocache
+    server 10.5.10.121 10.5.10.121:80 cookie s1 check port 80 send-proxy
+    server 10.5.10.122 10.5.10.122:80 cookie s2 check port 80 send-proxy
+	server 10.5.10.123 10.5.10.123:80 cookie s3 check port 80 send-proxy backup
+
+listen NGINX_https
+    bind *:443
+    mode tcp
+    timeout client 10800s
+    timeout server 10800s
+    # tcp-check expect string is\ running
+    balance leastconn
+    option tcp-check
+    # option allbackups
+    default-server port 9200 inter 2s downinter 5s rise 3 fall 2 slowstart 60s    maxconn 64 maxqueue 128 weight 100
+    cookie SERVERUSED insert indirect nocache
+    server 10.5.10.121 10.5.10.121:443 cookie s1 check port 443 send-proxy
+    server 10.5.10.122 10.5.10.122:443 cookie s2 check port 443 send-proxy
+	server 10.5.10.123 10.5.10.123:443 cookie s3 check port 443 send-proxy backup
+
+listen Gitea
+    bind *:3000
+    mode tcp
+    timeout client 10800s
+    timeout server 10800s
+    # tcp-check expect string is\ running
+    balance leastconn
+    option tcp-check
+    # option allbackups
+    default-server port 9200 inter 2s downinter 5s rise 3 fall 2 slowstart 60s    maxconn 64 maxqueue 128 weight 100
+    cookie SERVERUSED insert indirect nocache
+    server 10.5.10.121 10.5.10.121:3000 cookie s1 check port 3000
+    server 10.5.10.122 10.5.10.122:3000 cookie s2 check port 3000 backup
+	server 10.5.10.123 10.5.10.123:3000 cookie s3 check port 3000 backup
+
+listen Nextcloud
+    bind *:3001
+    mode tcp
+    timeout client 10800s
+    timeout server 10800s
+    # tcp-check expect string is\ running
+    balance leastconn
+    option tcp-check
+    # option allbackups
+    default-server port 9200 inter 2s downinter 5s rise 3 fall 2 slowstart 60s    maxconn 64 maxqueue 128 weight 100
+    cookie SERVERUSED insert indirect nocache
+    server 10.5.10.42 10.5.10.42:443 cookie s1 check port 443 send-proxy
+    server 10.5.10.43 10.5.10.43:443 cookie s2 check port 443 send-proxy backup
+
+listen Maxscale_rw
+    bind *:3309
+    mode tcp
+    timeout client 10800s
+    timeout server 10800s
+    tcp-check expect string is\ running
+    balance leastconn
+    option tcp-check
+    # option allbackups
+    default-server port 9200 inter 2s downinter 5s rise 3 fall 2 slowstart 60s    maxconn 64 maxqueue 128 weight 100
+    server 10.5.10.51 10.5.10.51:4008 check
+    server 10.5.10.52 10.5.10.52:4008 check backup
+    # server 10.5.10.53 10.5.10.53:4008 check backup

templates/keepalive.config

@@ -0,0 +1,32 @@
+#haproxy - You can add more types manually after this.
+        
+vrrp_script chk_haproxy {
+   script "killall -0 haproxy"   # verify the pid existance
+   interval 2                    # check every 2 seconds
+   weight 2                      # add 2 points of prio if OK
+}
+vrrp_instance VI_HAPROXY {
+   interface ens192                # interface to monitor
+   state MASTER
+   virtual_router_id 51          # Assign one ID for this route
+   priority 104                   
+   unicast_src_ip 10.5.10.51
+   unicast_peer {
+	  10.5.10.52
+	  10.5.10.53
+	  10.5.10.120
+	  10.5.10.121
+
+   }
+   virtual_ipaddress {
+       10.5.10.56                        # the virtual IP
+   } 
+   track_script {
+       chk_haproxy
+   }
+#    notify /usr/local/bin/notify_keepalived.sh
+}
+
+# DO NOT REMOVE THE NEXT LINE
+#@S9S_NEXT_SECTION@
+

templates/nginx.proxy.site

@@ -0,0 +1,31 @@
+server {
+	set $forward_scheme <<PROXY_SCHEME>>;
+	set $server         "<<PROXY_IP>>";
+	set $port           <<PROXY_PORT>>;
+
+	listen 80;
+	listen 443 ssl http2;
+
+	server_name <<SERVER_NAME>>;
+	
+	<<SECURE>>
+
+	include conf.d/include/letsencrypt-acme-challenge.conf;
+	include conf.d/include/ssl-ciphers.conf;
+	#ssl_certificate /opt/lb-data/letsencrypt/live/<<MAIN_SITE>>/fullchain.pem;
+	#ssl_certificate_key /opt/lb-data/letsencrypt/live/<<MAIN_SITE>>/privkey.pem;
+
+	access_log /var/log/nginx/<<MAIN_SITE>>.proxy.log proxy;
+	error_log /var/log/nginx/<<MAIN_SITE>>.error.log warn;
+	
+	<<EXPLOITS>>
+	<<WEBSOCKET>>
+	<<HSTS>>
+
+	location / {
+		<<WEBSOCKET>>
+		<<HSTS>>
+		include conf.d/include/force-ssl.conf;
+		include conf.d/include/proxy.conf;
+	}
+}

templates/nginx.site

@@ -0,0 +1,24 @@
+server {
+	listen 80;
+
+	server_name <<HOSTNAME>>;
+	set $base <<HOSTNAME>>;
+	root $base/public_html;
+
+	access_log  /var/log/nginx/<<HOSTNAME>>-access.log;
+	error_log  /var/log/nginx/<<HOSTNAME>>-error.log warn;
+
+	index index.php index.html index.htm;
+
+	location / {
+		try_files $uri $uri/ /index.php?$query_string;
+	}
+
+	location ~ \.php$ {
+		fastcgi_split_path_info ^(.+\.php)(/.+)$;
+		fastcgi_pass unix:/var/run/php/php8.2-fpm.sock;
+		include conf.d/include/php_fastcgi.conf;
+	}
+
+	include conf.d/include/general.conf;
+}

templates/nginx.ssl.site

@@ -0,0 +1,31 @@
+server {
+	listen 80;
+	listen 443 ssl http2;
+
+	server_name <<HOSTNAME>>;
+	set $base <<HOSTNAME>>;
+	root $base/public_html;
+
+	access_log  /var/log/nginx/<<HOSTNAME>>-access.log;
+	error_log  /var/log/nginx/<<HOSTNAME>>-error.log warn;
+
+	ssl_certificate /etc/letsencrypt/live/<<HOSTNAME>>/fullchain.pem;
+	ssl_certificate_key /etc/letsencrypt/live/<<HOSTNAME>>/privkey.pem;
+	include conf.d/include/ssl-ciphers.conf;
+
+	index index.php index.html index.htm;
+
+	location / {
+		try_files $uri $uri/ /index.php?$query_string;
+		include conf.d/include/force-ssl.conf;
+	}
+
+	location ~ \.php$ {
+		fastcgi_split_path_info ^(.+\.php)(/.+)$;
+		fastcgi_pass unix:/var/run/php/php8.2-fpm.sock;
+		include conf.d/include/php_fastcgi.conf;
+	}
+
+	include conf.d/include/general.conf;
+	include conf.d/include/letsencrypt-acme-challenge.conf;
+}

tmp-logrotate

@@ -0,0 +1,17 @@
+su root syslog
+
+/var/log/btmp {
+    missingok
+    daily
+    create 0660 root utmp
+    rotate 1
+	maxsize 100k
+}
+
+/var/log/wtmp {
+	size 10M
+	create 0664 root utmp
+	rotate 3
+	compress
+	daily
+}