From c5734df18ace9fa7c36e3ae3c16d80b4ba2803f6 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Sun, 30 Apr 2023 22:04:01 -0500 Subject: [PATCH] file split --- defaults.inc | 2 +- inc/certs.inc | 438 ++++++++++++ inc/sites.inc | 439 ++++++++++++ inc/status.inc | 682 +++++++++++++++++++ nodemgmt-scripts.sh | 1563 +------------------------------------------ 5 files changed, 1563 insertions(+), 1561 deletions(-) create mode 100755 inc/certs.inc create mode 100755 inc/sites.inc create mode 100755 inc/status.inc diff --git a/defaults.inc b/defaults.inc index a2075524..531c144a 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='4.6.7-04302023' +VERS='4.6.10-04302023' #NODETYPES=WEB,LB NODETYPES=MYSQL,LB,WEB,NC,WPD,LPD,MM,PW diff --git a/inc/certs.inc b/inc/certs.inc new file mode 100755 index 00000000..07b417fe --- /dev/null +++ b/inc/certs.inc @@ -0,0 +1,438 @@ +#!/usr/bin/env bash + +NEWCERT(){ + echo + if [ -z ${1+x} ]; then + echo -e -n "${idsCL[LightCyan]}Create certificate for what name (comma seperated for mutiple) : ${idsCL[Default]}" + read NEW_CERT + echo + else + NEW_CERT=${1} + fi + if [[ ${NEW_CERT} == *","* ]]; then + IFS=','; NEW_CERTS=(${NEW_CERT}); unset IFS + MAIN_CERT=${NEW_CERTS[0]} + else + MAIN_CERT=${NEW_CERT} + fi + if [ "${NEW_CERT}" != "" ]; then + echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}" + echo "" + + # echo -en "${idsCL[LightYellow]}Stopping other Webservers... ${idsCL[Default]}" + # for nip in "${WEB_HOSTS[@]}"; do + # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then + # # `ssh root@${nip} service nginx stop` + # # SERVICE nginx stop ${nip} + # fi + # done + # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" + # echo + + $CERT_DAEMON certonly --webroot -w /opt/lb-data/letsencrypt-acme-challenge -d ${NEW_CERT} + # $CERT_DAEMON certonly --dry-run --webroot -w /var/www/html -d ${NEW_CERT} + + chown -R root:le /opt/lb-data/letsencrypt + chmod -R 6775 /opt/lb-data/letsencrypt + + if [ -d /opt/lb-data/letsencrypt/live/${MAIN_CERT} ]; then + + touch /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert + if [ -f /opt/lb-data/nginx/sites-enabled/${MAIN_CERT}.conf ]; then + rm -f ${FOLDER}/cert-request.lastrun + daterun=`date +%Y-%m-%d-%H-%M-%S` + echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun + yes | cp -rfH ${FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun + DIVIDER true + echo -e -n "${idsCL[LightCyan]}Reload NGINX on LB Nodes (Y/n): ${idsCL[Default]}" + read -n 1 NGINXRELOAD + if [[ ${NGINXRELOAD} =~ ^[Nn]$ ]]; then + tmp='' + else + echo + echo -en "${idsCL[LightYellow]}Waiting for Cert replication across the nodes... ${idsCL[Default]}" + for nip in "${LB_HOSTS[@]}"; do + checkhost=$(CHECK_HOST ${nip}) + if [ "${checkhost}" != "false" ]; then + if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then + checked=false + until [ "${checked}" = "" ]; do + checked=`ssh root@${nip} "[ ! -f /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert ] && echo '.'"` + done + fi + fi + done + rm -f /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert + echo -e "${idsCL[Green]}Completed${idsCL[Default]}" + + echo + SERVICE nginx reload + fi + fi + echo + echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}" + else + echo + echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}" + fi + + # echo + # echo -en "${idsCL[LightYellow]}Starting other Webservers... ${idsCL[Default]}" + # for nip in "${WEB_HOSTS[@]}"; do + # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then + # `ssh root@${nip} service nginx start` + # # SERVICE nginx start ${nip} + # fi + # done + # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" + + echo "" + if [ -z $action ] || [ "${action}" = "gui" ]; then + DIVIDER true + ENTER2CONTINUE + fi + else + echo "You havent entered a site address." + sleep 1 + NEWCERT + exit 0 + fi +} + +DEL-SSL(){ + if [ -z ${1+x} ]; then + echo -e -n "${idsCL[LightCyan]}Delete what SSL site address: ${idsCL[Default]}" + read DEL_SSL + echo "" + else + DEL_SSL=${1} + fi + if [ ! -z ${DEL_SSL+x} ] && [ "${DEL_SSL}" != "" ]; then + if [ -d /etc/letsencrypt/live/${DEL_SSL} ]; then + echo -e "${idsCL[LightRed]}Deleting the SSL certificates for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}'...${idsCL[Default]}" + echo "" + + echo -en "${idsCL[LightRed]}Removing Files and Folders... ${idsCL[Default]}" + rm -rf /etc/letsencrypt/archive/${DEL_SSL} + rm -rf /etc/letsencrypt/live/${DEL_SSL} + rm -f /etc/letsencrypt/renewal/${DEL_SSL}.conf + echo -e "${idsCL[Green]}OK${idsCL[Default]}" + echo "" + + + else + echo -e "${idsCL[LightRed]}The SSL files folder for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}' could not be found.${idsCL[Default]}" + exit 1 + fi + if [ "${timeout}" != "true" ]; then + echo -e "${idsCL[Green]}Completed${idsCL[Default]}" + fi + echo + # SERVICE nginx reload + echo -e "${idsCL[LightRed]}The SSL certificate has been removed fromt be nodes.${idsCL[Default]}" + fi +} + +CERTRENEW(){ + echo -en "${idsCL[LightCyan]}Stopping Webserver-Node2...${idsCL[Default]}" + ssh root@webserver-node2.scity.us service nginx stop + echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}" + echo + echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}" + echo + sleep 5 + $CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun + # $CERT_DAEMON renew --force-renewal --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun + # $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun + CONCAT_SSL + chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun + chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun + yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun + daterun=`date +%Y-%m-%d-%H-%M-%S` + echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun + DIVIDER true + echo -en "${idsCL[LightCyan]}Starting Webserver-Node2 Back up...${idsCL[Default]}" + ssh root@webserver-node2.scity.us service nginx start + echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}" + echo + SERVICE nginx reload 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun + echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}" + echo "" + if [ -z $action ] || [ "${action}" = "gui" ]; then + DIVIDER true + ENTER2CONTINUE + fi +} +NIGHTLYRENEW(){ + rm -f ${FOLDER}/cert-renewal.lastrun + ssh root@webserver-node2.scity.us service nginx stop + sleep 5 + $CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun + CONCAT_SSL + chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun + chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun + yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun + daterun=`date +%Y-%m-%d-%H-%M-%S` + echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun + ssh root@webserver-node2.scity.us service nginx start + SERVICE nginx reload web &>> ${FOLDER}/cert-renewal.lastrun +} + +CONCAT_SSL(){ + rm -f /tmp/ssllist + for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist + for certdir in $( ${certdir}fullcert.pem + done +} + +LISTCERTS(){ + declare -i cw; declare -i spc1; declare -i c + declare -A CHECKCERT_DOMAINS + IFS=$'\n' + for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do + HOST=${LINE%% *} + PORT=${LINE#* } + IFS=" " + CHECKCERT_DOMAINS[${HOST}]=${PORT} + done + unset IFS + if [ ! -z ${LOCAL_SERVICES+x} ]; then + NCMD="ssh root@${MYSQL_HOSTS[0]}" + #${NCMD} rm -f /tmp/ssllist + #${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist + else + NCMD='' + rm -f /tmp/ssllist + #for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist + fi + if [ -z $action ] || [ "${action}" = "gui" ]; then + DIVIDER true + fi + echo + echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}" + DIVIDER false yellow 120 + echo -e "Subject Name Monitored Expires Alternate Subject Names" + DIVIDER false yellow 120 + + cw=30; + c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done + + #ssldir=$(${NCMD} find /opt/nginx-proxy/ssl/* -type l) + ssldir=$(${NCMD} find /opt/lb-data/letsencrypt/live/* -type d) + for certdir in ${ssldir[@]}; do + SUBJECT=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV) + SUBJECTNAMES=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV) + CERTEXPIRE=$(date -d "$(: | ${NCMD} openssl x509 -in ${certdir}/cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s'); + + SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, } + # SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/\n/, /g") + SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g") + SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g") + SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g") + IFS=', '; SUBJECT_NAMES=(${SUBJECTNAMES}); unset IFS + DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS))); + + c=0; spc='' + spc1=${cw}-${#SUBJECT} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + if [ $CERTEXPIRE -le $DUEIN ]; then + date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}" + SENDNOTICE "SSL Expiring" "${SUBJECT} expires on ${date}" 1 + else + date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}" + fi + + if [ "${CHECKCERT_DOMAINS[${SUBJECT}]}" = "" ]; then + monitored='No ' + else + monitored="${idsCL[Green]}Yes${idsCL[Default]}" + fi + if [ ${#SUBJECT_NAMES[@]} -lt 4 ]; then + echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored} ${date} ${SUBJECTNAMES}" + elif [ ${#SUBJECT_NAMES[@]} -lt 7 ]; then + echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored} ${date} ${SUBJECT_NAMES[0]}, ${SUBJECT_NAMES[1]}, ${SUBJECT_NAMES[2]}" + echo -e "${spc2} ${SUBJECT_NAMES[3]}, ${SUBJECT_NAMES[4]}, ${SUBJECT_NAMES[5]}" + + fi + DIVIDER false darkGray 120 + done + echo + if [ -z $action ] || [ "${action}" = "gui" ]; then + DIVIDER true + ENTER2CONTINUE + fi + echo "" +} + +LISTCERTS_NPM(){ + declare -i cw; declare -i spc1; declare -i c + declare -A CHECKCERT_DOMAINS + IFS=$'\n' + for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do + HOST=${LINE%% *} + PORT=${LINE#* } + IFS=" " + CHECKCERT_DOMAINS[${HOST}]=${PORT} + done + unset IFS + if [ ! -z ${LOCAL_SERVICES+x} ]; then + NCMD="ssh root@${MYSQL_HOSTS[0]}" + ${NCMD} rm -f /tmp/ssllist + ${NCMD} 'for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist + else + NCMD='' + rm -f /tmp/ssllist + for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist + fi + if [ -z $action ] || [ "${action}" = "gui" ]; then + DIVIDER true + fi + echo + echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}" + DIVIDER false yellow 120 + echo -e "Subject Name Monitored Expires Alternate Subject Names" + DIVIDER false yellow 120 + + cw=30; + c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done + for certdir in $( ${FOLDER}/ssl-cert-check.lastrun + SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)" + SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG" + else + # ${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains + IFS=$'\n' + for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do + HOST=${LINE%% *} + PORT=${LINE#* } + CERTINFO=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p ${PORT} -s ${HOST} -N) + CERTVALID=$(echo ${CERTINFO} | awk -F' valid' '{print $1}') + CERTVALID=${CERTVALID: -1} + CERTDAYS=${CERTINFO#*days=} + if [ "${CERTVALID}" == "0" ] || [ "${CERTVALID}" == "E" ]; then + printf "%-30s: %s\n" "${HOST}" "SSL is not valid" + SENDNOTICE "SSL Not Valid" "The (${HOST}) SSL is not valid" 1 + + elif [ ${CERTDAYS} -lt 1 ]; then + printf "%-30s: %s\n" "${HOST}" "SSL Expired! (${CERTDAYS} days)" + SENDNOTICE "SSL Expired" "The (${HOST}) SSL is expired!! (${CERTDAYS} days)" 2 + + elif [ ${CERTDAYS} -lt 15 ]; then + printf "%-30s: %s\n" "${HOST}" "SSL is expiring in $CERTDAYS days!" + SENDNOTICE "SSL Expiring" "The (${HOST}) SSL is expiring in ${CERTDAYS} days" 1 + + else + printf "%-30s: %s\n" "${HOST}" "SSL is valid for ${CERTDAYS} days" + fi + done + unset IFS + fi +} \ No newline at end of file diff --git a/inc/sites.inc b/inc/sites.inc new file mode 100755 index 00000000..20bb41d5 --- /dev/null +++ b/inc/sites.inc @@ -0,0 +1,439 @@ +#!/usr/bin/env bash + +DELSITE(){ + while [ $# -gt 0 ]; do + case "$1" in + -site) DEL_SITE=${2};; + -ssl) DEL_SSL=${2};; + -list) DELSITES; exit 0;; + -*) + echo "Invalid option: '${1}' requires an argument" 1>&2 + echo "" + echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {" + width=33 + printf "%-${width}s- %s\n" " -site {FQDN address}" "(*required)" + printf "%-${width}s- %s\n" " -ssl {yes or [no]}" "Delete SSL certs as well" + printf "%-${width}s- %s\n" " -list" "List sites (same as running nodemgmt delsites)" + echo "}" + exit 1;; + esac + shift + done + if [ -z ${DEL_SITE+x} ]; then + echo -e -n "${idsCL[LightCyan]}Delete what site address: ${idsCL[Default]}" + read DEL_SITE + echo "" + fi + if [[ $DEL_SSL =~ ^[Nn]$ ]]; then + DEL_SSL=no + elif [[ $DEL_SSL =~ ^[Yy]$ ]]; then + DEL_SSL=yes + elif [ -z ${DEL_SSL+x} ]; then + echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${DEL_SITE}' as well? [y/N]${idsCL[Default]} " + read DEL_SSL + fi + if [ ! -z ${DEL_SITE+x} ] && [ "${DEL_SITE}" != "" ]; then + echo -e "${idsCL[LightRed]}Deleting site '${idsCL[Red]}${DEL_SITE^^}${idsCL[LightRed]}'...${idsCL[Default]}" + echo "" + + echo -e "${idsCL[LightRed]}[[Removing Files and Folders]]${idsCL[Default]}" + echo -e "${idsCL[LightRed]}-------------------------------------------${idsCL[Default]}" + echo + echo -en "${idsCL[LightCyan]}Removing files from all Nodes ... ${idsCL[Default]}" + ssh root@10.10.1.120 rm -f /etc/nginx/sites-enabled/${DEL_SITE}* >/dev/null 2>&1 + ssh root@10.10.10.80 rm -f /etc/nginx/sites-enabled/${DEL_SITE}* >/dev/null 2>&1 + if [ "${DEL_SSL}" == "yes" ]; then + ssh root@10.10.10.80 rm -rf /etc/letsencrypt/archive/${DEL_SITE} >/dev/null 2>&1 + ssh root@10.10.10.80 rm -rf /etc/letsencrypt/live/${DEL_SITE} >/dev/null 2>&1 + ssh root@10.10.10.80 rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf >/dev/null 2>&1 + fi + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + echo + echo -en "${idsCL[LightCyan]}Reloading NGINX ... ${idsCL[Default]}" + ssh root@10.10.10.80 nodemgmt service nginx reload >/dev/null 2>&1 + ssh root@10.10.1.120 nodemgmt service nginx reload >/dev/null 2>&1 + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + + # nid=1 + # for nip in "${WEB_HOSTS[@]}"; do + # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then + # nip='localhost ' + # NCMD='' + # else + # NCMD="ssh root@${nip}" + # fi + # echo -en "Removing from Webserver-Node${nid} ($nip)... ${idsCL[Default]}" + # if [ "${NCMD}" != "" ]; then + # checkhost=$(CHECK_HOST ${nip}) + # fi + # if [ "${checkhost}" != "false" ]; then + # #if [ -f /etc/nginx/sites-available/${DEL_SITE} ]; then + # ${NCMD} rm -f /etc/nginx/sites-available/${DEL_SITE}* + # ${NCMD} rm -f /etc/nginx/sites-enabled/${DEL_SITE}* + # #fi + # #if [ -d /var/www/${DEL_SITE} ]; then + # ${NCMD} rm -rf /var/www/${DEL_SITE} + # #fi + # if [ "${DEL_SSL}" = "yes" ]; then + # ${NCMD} rm -rf /etc/letsencrypt/archive/${DEL_SITE} + # ${NCMD} rm -rf /etc/letsencrypt/live/${DEL_SITE} + # ${NCMD} rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf + # fi + # echo -e "${idsCL[Green]}OK${idsCL[Default]}" + # else + # echo -e "${idsCL[Red]}Node is down${idsCL[Default]}" + # fi + # nid=`expr $nid + 1` + # done + # echo "" + # SERVICE nginx reload ns + echo -e "${idsCL[LightRed]}Site has been deleted.${idsCL[Default]}" + else + echo "Missing arguments" + echo "" + echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {" + width=33 + printf "%-${width}s- %s\n" " -site {FQDN address}" "Site to delete" + printf "%-${width}s- %s\n" " -ssl {yes or [no]}" "Delete SSL certs as well" + printf "%-${width}s- %s\n" " -list" "List sites (same as running nodemgmt delsites)" + echo "}" + exit 1 + fi +} + +DELSITES(){ + echo + echo -e "${idsCL[Red]}Select a site to delete...${idsCL[Default]}" + DIVIDER true + sid=1 + filels="( $(ssh root@${WEB_HOSTS[0]} ls '/etc/nginx/sites-available/*') )" + # IFS='\n' + for siteconf in $filels; do + # for siteconf in /etc/nginx/sites-available/* ; do + # [ -e "$siteconf" ] || continue + if [ ${siteconf:0:1} == '/' ]; then + IFS='/'; site_conf=(${siteconf}); unset IFS + SITES[${sid}]=${site_conf[4]} + sid=`expr $sid + 1` + fi + done + for s in "${!SITES[@]}"; do + echo " [${s}] ${SITES[${s}]}" + done + echo "" + if [ -z $action ] || [ "${action}" = "gui" ]; then + echo " [B] Back" + fi + echo " [Q] Quit" + echo "" + echo -e -n "${idsCL[LightYellow]}Please select a site from above from above:${idsCL[Default]} " + read selsite + echo "" + if [ -z ${SITES[$selsite]} ] && [ "${selsite}" != "Q" ] && [ "${selsite}" != "q" ] && [ "${selsite}" != "B" ] && [ "${selsite}" != "b" ]; then + echo "Thats an invaild option," + echo "please select a valid option only." + sleep 1 + DELSITES + exit 0 + elif [ "${selsite}" = "Q" ] || [ "${selsite}" = "q" ]; then + exit 0 + elif [ "${selsite}" = "B" ] || [ "${selsite}" = "b" ]; then + GUI + else + while : + do + echo -e -n "${idsCL[LightRed]}Are you sure you want to delete '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}'? [y/N]${idsCL[Default]} " + read response + echo "" + if [[ $response =~ ^[Yy]$ ]]; then + echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}', if they exist? [y/N]${idsCL[Default]} " + read sslresponse + DELSITE -site ${SITES[${selsite}]} -ssl ${sslresponse} + echo "" + DIVIDER + ENTER2CONTINUE + break + else + break + fi + done + DELSITES + exit 0 + fi + if [ -z $action ] || [ "${action}" = "gui" ]; then + ENTER2CONTINUE + fi +} + +NEWSITE(){ + echo + while [ $# -gt 0 ]; do + case "$1" in + -site) NEW_SITE=${2};; + -type) SITE_TYPE=${2};; + -ssl) CREATE_SSL=${2};; + -proxy_scheme) PROXYSCHEME=${2};; + -proxy_host) PROXYHOST=${2};; + -proxy_port) PROXYPORT=${2};; + -websocket) WEBSOCKET=${2};; + -hsts) HSTS=${2};; + -exploits) EXPLOITS=${2};; + -secure) SECURE=${2};; + -h | -help | --help) + echo "" + echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {" + width=33 + printf "%-${width}s- %s\n" " -site {FQDN address(,es)}" "(new site and aliases, comma separated)" + printf "%-${width}s- %s\n" " -ssl {yes or no}" "(defaults to yes)" + printf "%-${width}s- %s\n" " -type {'local' or 'proxy'}" "(defaults to local)" + printf "%-${width}s- %s\n" " -proxy_port {host port}" "(proxy backend host)" + printf "%-${width}s- %s\n" " -proxy_host {IP or FQDN}" "(proxy backend port)" + printf "%-${width}s- %s\n" " -proxy_scheme {http or https}" "(proxy backend scheme)" + printf "%-${width}s- %s\n" " -websocket {yes or no}" "(websocket support)" + printf "%-${width}s- %s\n" " -hsts {yes or no}" "(hsts support)" + printf "%-${width}s- %s\n" " -exploits {yes or no}" "(block exploits)" + printf "%-${width}s- %s\n" " -secure {yes or no}" "(secure access [nginx/.htpasswd])" + echo "}" + exit 1;; + esac + shift + done + + #if [ -z ${SITE_TYPE+x} ]; then SITE_TYPE=local; fi + #if [ -z ${CREATE_SSL+x} ]; then CREATE_SSL=true; fi + if [ -z ${NEW_SITE+x} ]; then + echo -e -n "${idsCL[LightCyan]}New site domain name (comma seperated for multiple) : ${idsCL[Default]}" + read NEW_SITE + showdivide=yes + fi + + if [ -z ${CREATE_SSL+x} ]; then + echo -e -n "${idsCL[LightCyan]}Create SSL for site? [Y/n] ${idsCL[Default]}" + read CREATE_SSL + showdivide=yes + if [[ $CREATE_SSL =~ ^[Yy]$ ]] || [ "${CREATE_SSL}" = "" ]; then + CREATE_SSL=yes + # echo -e -n "${idsCL[LightCyan]}Add additonal domain names to the SSL cert (comma seperated)? : ${idsCL[Default]}" + # read ssladd + else + CREATE_SSL=no + fi + fi + + if [ -z ${SITE_TYPE+x} ]; then + echo -e -n "${idsCL[LightCyan]}Site type (local/{proxy}): ${idsCL[Default]}" + read SITE_TYPE + showdivide=yes + if [ "${SITE_TYPE}" = "" ]; then + SITE_TYPE=proxy + fi + fi + if [ "${SITE_TYPE}" = "proxy" ]; then + if [ -z ${PROXYHOST+x} ]; then + echo -e -n "${idsCL[LightCyan]}What is the proxy backend address (IP or FQDN): ${idsCL[Default]}" + read PROXYHOST + showdivide=yes + fi + if [ -z ${PROXYPORT+x} ]; then + echo -e -n "${idsCL[LightCyan]}What is the proxy backend port (tcp port): ${idsCL[Default]}" + read PROXYPORT + showdivide=yes + fi + if [ -z ${PROXYSCHEME+x} ]; then + echo -e -n "${idsCL[LightCyan]}What is the proxy backend scheme (http/https): ${idsCL[Default]}" + read PROXYSCHEME + showdivide=yes + fi + if [ -z ${WEBSOCKET+x} ]; then + echo -e -n "${idsCL[LightCyan]}Enable Websocket Support (y/N): ${idsCL[Default]}" + read WEBSOCKET + showdivide=yes + if [[ ${WEBSOCKET} =~ ^[Nn]$ ]] || [ "${WEBSOCKET}" = "" ]; then + WEBSOCKET=no + elif [[ ${WEBSOCKET} =~ ^[Yy]$ ]]; then + WEBSOCKET=yes + else + WEBSOCKET=no + fi + fi + if [ -z ${HSTS+x} ]; then + echo -e -n "${idsCL[LightCyan]}Enable HSTS Support (Y/n): ${idsCL[Default]}" + read HSTS + showdivide=yes + [[ ${HSTS} =~ ^[Yy]$ ]] || [ "${HSTS}" = "" ] &&HSTS=yes || HSTS=no + fi + if [ -z ${EXPLOITS+x} ]; then + echo -e -n "${idsCL[LightCyan]}Block exploits (y/N): ${idsCL[Default]}" + read EXPLOITS + showdivide=yes + if [[ ${EXPLOITS} =~ ^[Nn]$ ]] || [ "${EXPLOITS}" = "" ]; then + EXPLOITS=no + elif [[ ${EXPLOITS} =~ ^[Yy]$ ]]; then + EXPLOITS=yes + else + EXPLOITS=no + fi + fi + if [ -z ${SECURE+x} ]; then + echo -e -n "${idsCL[LightCyan]}Secure site with passwords from [ nginx/.htpasswd ](y/N): ${idsCL[Default]}" + read SECURE + showdivide=yes + if [[ ${SECURE} =~ ^[Nn]$ ]] || [ "${SECURE}" = "" ]; then + SECURE=no + elif [[ ${SECURE} =~ ^[Yy]$ ]]; then + SECURE=yes + else + SECURE=no + fi + fi + fi + [ "${showdivide}" == "yes" ] && DIVIDER + echo + width=14 + printf "%-${width}s: %s\n" "New site" "${NEW_SITE}" + printf "%-${width}s: %s\n" "Create SSL" "${CREATE_SSL}" + printf "%-${width}s: %s\n" "Site type" "${SITE_TYPE}" + if [ "${SITE_TYPE}" = "proxy" ]; then + printf "%-${width}s: %s\n" "Proxy host" "${PROXYHOST}" + printf "%-${width}s: %s\n" "Proxy port" "${PROXYPORT}" + printf "%-${width}s: %s\n" "Proxy scheme" "${PROXYSCHEME}" + printf "%-${width}s: %s\n" "Websocket Support" "${WEBSOCKET}" + printf "%-${width}s: %s\n" "HSTS Support" "${HSTS}" + printf "%-${width}s: %s\n" "Block Exploits" "${EXPLOITS}" + printf "%-${width}s: %s\n" "Secure Access" "${SECURE}" + fi + echo -e -n "${idsCL[LightRed]}Is this information correct? [Y/n]${idsCL[Default]} " + read -n 1 response + echo + if [[ $response =~ ^[Yy]$ ]] || [ "${response}" = "" ]; then + if [[ ${NEW_SITE} == *","* ]]; then + IFS=','; NEW_SITES=(${NEW_SITE}); unset IFS + MAIN_SITE=${NEW_SITES[0]} + NGINX_SERVERNAME=${NEW_SITE//[,]/ } + else + MAIN_SITE=${NEW_SITE} + NGINX_SERVERNAME=${NEW_SITE} + + fi + if [ "${SITE_TYPE}" = "proxy" ]; then + if [ ! -z ${PROXYSCHEME+x} ] && [ ! -z ${PROXYHOST+x} ] && [ ! -z ${PROXYPORT+x} ]; then GO=true; fi + else GO=true + fi + if [ "${GO}" = "true" ]; then + echo -e "${idsCL[LightGreen]}Setting up new site for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}" + echo "" + + [ "${WEBSOCKET}" == "yes" ] && WEBSOCKET="include conf.d\/include\/websocket-support.conf;" || WEBSOCKET="" + [ "${HSTS}" == "yes" ] && HSTS="include conf.d\/include\/hsts-support.conf;" || HSTS="" + [ "${EXPLOITS}" == "yes" ] && EXPLOITS="include conf.d\/include\/block-exploits.conf;" || EXPLOITS="" + [ "${SECURE}" == "yes" ] && SECURE="include conf.d\/include\/secure-access.conf;" || SECURE="" + + + ######################################### LOCAL + if [ "${SITE_TYPE}" = "local" ]; then + echo -e "server { + listen 80;" > /etc/nginx/sites-available/${MAIN_SITE} + if [ "${CREATE_SSL}" = "yes" ]; then + echo -e " listen 443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE} + fi + echo -e " + server_name ${NGINX_SERVERNAME}; + + set \$base /var/www/${MAIN_SITE}; + root \$base/public_html; + + access_log /var/log/nginx/${MAIN_SITE}-access.log; + error_log /var/log/nginx/${MAIN_SITE}-error.log warn;" >> /etc/nginx/sites-available/${MAIN_SITE} + + if [ "${CREATE_SSL}" = "yes" ]; then + echo -e " + ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem; + include conf.d/include/ssl-ciphers.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} + fi + echo -e " + index index.php; + + location / { + try_files \$uri \$uri/ /index.php?\$query_string;" >> /etc/nginx/sites-available/${MAIN_SITE} + if [ "${CREATE_SSL}" = "yes" ]; then + echo -e " include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} + fi + echo -e " } + + location ~ \.php\$ { + fastcgi_pass unix:/var/run/php/php7.2-fpm.sock; + include conf.d/include/php_fastcgi.conf; + } + + include conf.d/include/general.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} + if [ "${CREATE_SSL}" = "yes" ]; then + echo -e " include conf.d/include/letsencrypt-acme-challenge.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} + fi + echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE} + + + sudo -u www-data mkdir -p /var/www/${MAIN_SITE}/{public_html,nginx_logs} + # echo -en "${idsCL[LightYellow]}Waiting for folder replication across the webserver nodes... ${idsCL[Default]}" + # for nip in "${WEB_HOSTS[@]}"; do + # checkhost=$(CHECK_HOST ${nip}) + # if [ "${checkhost}" != "false" ]; then + # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then + # checked=false + # until [ "${checked}" = "" ]; do + # checked=`ssh root@${nip} "[ ! -d /var/www/${MAIN_SITE} ] && echo does not exist"` + # done + # fi + # fi + # done + # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" + # + # echo -en "${idsCL[LightYellow]}Setting folder permissions... ${idsCL[Default]}" + # SET-PERMISSIONS ${MAIN_SITE} + # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" + + + + ######################################### PROXY + else + + cp ${FOLDER}/templates/nginx.proxy.site /etc/nginx/sites-enabled/${MAIN_SITE}.conf + sed -i "s/<>/${NGINX_SERVERNAME}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + sed -i "s/<>/${MAIN_SITE}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + sed -i "s/<>/${PROXYHOST}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + sed -i "s/<>/${PROXYPORT}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + sed -i "s/<>/${PROXYSCHEME}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + sed -i "s/<>/${WEBSOCKET}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + sed -i "s/<>/${HSTS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + sed -i "s/<>/${EXPLOITS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + sed -i "s/<>/${SECURE}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + fi + + if [ "${CREATE_SSL}" = "yes" ]; then + [ "${ssladd}" != "" ] && NEWCERT ${NEW_SITE},${ssladd} || NEWCERT ${NEW_SITE} + if [ "${SITE_TYPE}" == "proxy" ]; then + sed -i "s/#ssl_certificate/ssl_certificate/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf + fi + fi + + rm -f ${FOLDER}/new-site.lastrun + daterun=`date +%Y-%m-%d-%H-%M-%S` + echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun + # yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun + # yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun + # daterun=`date +%Y-%m-%d-%H-%M-%S` + # echo -e "${daterun}" >> /etc/nginx/new-site.lastrun + DIVIDER true + echo "" + echo -e "${idsCL[LightGreen]}The new site for '${idsCL[LightGreen]}${NEW_SITE}${idsCL[Default]}' has been created.${idsCL[Default]}" + echo "" + else + echo "Missing proxy arguments" + Exit 1 + fi + + else + ${SCRIPT} newsite + exit 0 + fi +} \ No newline at end of file diff --git a/inc/status.inc b/inc/status.inc new file mode 100755 index 00000000..baa6f19e --- /dev/null +++ b/inc/status.inc @@ -0,0 +1,682 @@ +#!/usr/bin/env bash + +STATUS(){ + start=`date +%s` + ST_ACTION=${1} + if [ "${ST_ACTION}" = "report" ]; then + unset idsCL idsBG idsST + idsCL=('') + idsBG=('') + idsST=('') + elif [ "${ST_ACTION}" = "sync" ]; then + ST_ACTION=repl + fi + declare -i cw; declare -i spc1; declare -i c + + ######################## + if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}" + fi + for NTYPE in "${NODE_TYPES[@]}"; do + PH=${NTYPE}_HOSTS[0] + if [ ! -f ${FOLDER}/${!PH}.down ]; then + PH_CMD="ssh root@${!PH}" + var=${NTYPE}_REPL_CHECK[@] + if [ ! -z ${!var+x} ]; then + for rcheck in "${!var}"; do + # echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}" + ${PH_CMD} rm -f ${FOLDER}/test.repl + daterun=`date +%Y-%m-%d-%H-%M-%S` + ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" & + done + fi + fi + done + replstart=`date +%s` + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Complete${idsCL[Default]}" + echo + fi + fi + + ######################## + if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then + if [ ! -z ${LOCAL_SERVICES+x} ]; then + if [ "${ST_ACTION}" != "check" ]; then + lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) + uptime=`uptime -p` + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" + DIVIDER false green + fi + for srvc in "${LOCAL_SERVICES[@]}"; do + if [ "${ST_ACTION}" != "check" ]; then + if [ "${ST_ACTION}" != "report" ]; then + c=0; cw=30; spc='' + spc1=${cw}-${#NM_SERVICES[${srvc}]} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + else + spc=' ' + fi + echo -en " ${NM_SERVICES[${srvc}]}${spc}: " + fi + + if [ "$(systemctl is-active ${srvc})" != "active" ]; then + if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + fi + SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1 + touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down + echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then + mv ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down + fi + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi + if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then + if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) + else + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down) + fi + SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.* + echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE} + + fi + fi + done + + if [ "${ST_ACTION}" != "check" ]; then echo; fi + fi + ######################## + + for NTYPE in "${NODE_TYPES[@]}"; do + nid=1 + + dockers=${NTYPE}_DOCKER[@] + hosts=${NTYPE}_HOSTS[@] + + var=${NTYPE}_HOSTS[@] + + if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" == "dockers" ]; then + GOFORCHECK=false; + else + GOFORCHECK=true; + fi + + if [ ${GOFORCHECK} = true ]; then + + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsST[Bold]}"; DIVIDER + if [ "${ST_ACTION}" = "dockers" ]; then + echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" + fi + DIVIDER; echo -e "${idsST[Reset]}" + fi + for nip in "${!var}"; do + [ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}" + + if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' + else NCMD="ssh root@${nip}"; LH='' + fi + if [ "${NCMD}" != "" ]; then + checkhost=$(CHECK_HOST ${nip}) + fi + if [ "${checkhost}" != "false" ]; then + if [ "${ST_ACTION}" != "check" ]; then + #echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}" + if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then + uptime=`${NCMD} uptime -p` + echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" + else + echo -e "${idsCL[Default]}" + fi + + DIVIDER false green + fi + if [ -f ${FOLDER}/${nip}.down ]; then + if [ -f ${FOLDER}/${nip}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + else + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + fi + rm -f ${FOLDER}/${nip}.* + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + fi + + if [ "${ST_ACTION}" != "dockers" ]; then + srvcs=${NTYPE}_SERVICES_CHECK[@]; + srvcstotest="$(join_by " " ${!srvcs})" + srvctst=(`${NCMD} systemctl is-active ${srvcstotest}`) + sr=0 + for srvc in "${!srvcs}"; do + [ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false || NOGOCHK=true + if [ ${NOGOCHK} == true ]; then + if [ "${ST_ACTION}" != "check" ]; then + if [ "${ST_ACTION}" != "report" ]; then + c=0; cw=30; spc=''; spc1=${cw}-${#NM_SERVICES[${srvc}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + else + spc=' ' + fi + echo -en " ${NM_SERVICES[${srvc}]}$spc: " + fi + if [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" == "mysql" ]; then + [ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false + elif [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" = "mariadb" ]; then + [ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false + elif [ "${srvctst[$sr]}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then + mysqlgo=true + fi + # echo "mysqlgo=${mysqlgo}" + if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then + [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 + touch ${FOLDER}/${nip}-${srvc}.down + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ] && mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + touch ${FOLDER}/${nip}-${srvc}.down + # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & + fi + [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + else + [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" + if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then + [ -f ${FOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} + + rm -f ${FOLDER}/${nip}-${srvc}.* + + fi + # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then + if [ "${srvc}" == "gitea" ]; then + rm -f ${FOLDER}/*-${srvc}.* + fi + fi + [ "${mysqlgo}" = "true" ] && unset mysqlgo + fi + sr=`expr $sr + 1` + done + + + + + + # for srvc in "${!var2}"; do + # NOGOCHK=true; + # [ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false; + # if [ ${NOGOCHK} = true ]; then + # if [ "${ST_ACTION}" != "check" ]; then + # if [ "${ST_ACTION}" != "report" ]; then + # c=0; cw=30; spc='' + # spc1=${cw}-${#NM_SERVICES[${srvc}]} + # until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + # else + # spc=' ' + # fi + # echo -en " ${NM_SERVICES[${srvc}]}$spc: " + # fi + # srvctst=$(${NCMD} systemctl is-active ${srvc}) + # if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then + # mysqlgo=false; + # [ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true; + # elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then + # mysqlgo=false; + # [ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true; + # elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then + # mysqlgo=true + # fi + # # echo "mysqlgo=${mysqlgo}" + # if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then + # if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then + # if [ "${ST_ACTION}" != "check" ]; then + # echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + # fi + # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 + # touch ${FOLDER}/${nip}-${srvc}.down + # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} + # + # else + # errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + # if [ $errtime -gt ${RENOTIFY} ]; then + # if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + # mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + # fi + # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + # touch ${FOLDER}/${nip}-${srvc}.down + # if [ "${srvc}" = "mysql" ]; then + # ${NCMD} systemctl restart ${srvc} + # fi + # fi + # if [ "${ST_ACTION}" != "check" ]; then + # echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + # fi + # fi + # else + # if [ "${ST_ACTION}" != "check" ]; then + # echo -e "${idsCL[Green]}Running${idsCL[Default]}" + # fi + # if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then + # if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + # else + # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + # fi + # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} + # + # rm -f ${FOLDER}/${nip}-${srvc}.down + # rm -f ${FOLDER}/${nip}-${srvc}.errtime + # + # fi + # # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then + # if [ "${srvc}" == "gitea" ]; then + # rm -f ${FOLDER}/*-${srvc}.down + # rm -f ${FOLDER}/*-${srvc}.errtime + # fi + # fi + # [ "${mysqlgo}" = "true" ] && unset mysqlgo + # fi + # done + fi + + if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo + echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}" + echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}" + fi + + for docker in "${!dockers}"; do + if [ "${ST_ACTION}" != "check" ]; then + if [ "${ST_ACTION}" != "report" ]; then + c=0; cw=30; spc='' + spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}` + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + else + spc=' ' + fi + echo -en " ${NM_DOCKERS[${docker}]}$spc: " + fi + + if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then + if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then + ${NCMD} docker rm ${docker} >/dev/null 2>&1 + fi + ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 + sleep 10s + if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" + fi + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed" + rm -f ${FOLDER}/${nip}-${docker}.down + rm -f ${FOLDER}/${nip}-${docker}.errtime + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE} + + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" + + if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1 + touch ${FOLDER}/${nip}-${docker}.down + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE} + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then + mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime) + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + touch ${FOLDER}/${nip}-${docker}.down + fi + fi + + fi + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi + rm -f ${FOLDER}/${nip}-${docker}.down + rm -f ${FOLDER}/${nip}-${docker}.errtime + fi + done + fi + + else + if [ ! -f ${FOLDER}/${nip}.down ]; then + touch ${FOLDER}/${nip}.down + if [ ! -f ${FOLDER}/${nip}.errtime ]; then + touch ${FOLDER}/${nip}.errtime + fi + if [ "${ST_ACTION}" != "check" ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" + fi + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1 + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE} + else + if [ "${ST_ACTION}" != "check" ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" + fi + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + # if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then + # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE} + # fi + fi + + fi + if [ "${ST_ACTION}" != "check" ]; then echo; fi + nid=`expr $nid + 1` + done + + fi #GOFORCHECK + + done + + fi + ######################## + ## REPLICATION CHECK + ######################## + + if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsST[Bold]}"; DIVIDER + echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" + DIVIDER; echo -e "${idsST[Reset]}" + fi + replcheckstart=`date +%s` + echo "$(date) - STARTING REPL CHECK - $(SHOW_TIME $(expr `date +%s` - $replstart) s)" >> ${FOLDER}/log.repl + for NTYPE in "${NODE_TYPES[@]}"; do + PH=${NTYPE}_HOSTS[0] + PH_CMD="ssh root@${!PH}" + var=${NTYPE}_REPL_CHECK[@] + if [ ! -z ${!var+x} ]; then + nid=1 + + var=${NTYPE}_HOSTS[@] + for nip in "${!var}"; do + + # echo "CHECK: ${NTYPE} - ${nip}" + + if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" + DIVIDER false green + fi + + # checkhost=$(CHECK_HOST ${nip}) + #if [ "${checkhost}" != "false" ]; then + if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then + var2=${NTYPE}_REPL_CHECK[@] + # timeout=`date --date='20 seconds' +%s` + for rcheck in "${!var2}"; do + if [ "${ST_ACTION}" != "check" ]; then + if [ "${ST_ACTION}" != "report" ]; then + c=0; cw=30; spc='' + spc1=${cw}-${#REPL_DESC[${rcheck}]} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + else + spc=' ' + fi + echo -en " ${REPL_DESC[${rcheck}]}${spc}: " + + timeout=`date --date='2 minutes' +%s` + + # if [ "${rcheck}" == "www" ]; then + # timeout=`date --date='2 minutes' +%s` + # else + # timeout=`date --date='20 seconds' +%s` + # fi + + else + timeout=`date --date='2 minutes' +%s` + fi + warn_timeout=`date --date='30 secs' +%s` + + checked=false + until [ "${checked}" = "" ]; do + if [ "${PH_CMD}" = "" ]; then + ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` + else + ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" + fi + if [ "`date +%s`" -gt "${timeout}" ]; then + timeout=true + break + elif [ "`date +%s`" -gt "${warn_timeout}" ]; then + echo -en " waiting for sync... " + warn_timeout=`date --date='100 minutes' +%s` + fi + done + replend=`date +%s` + replruntime=$((replend-replstart)) + replcheckruntime=$((replend-replcheckstart)) + if [ "${timeout}" = "true" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Timeout${idsCL[Default]} - $(SHOW_TIME ${replruntime} s) since copy & $(SHOW_TIME ${replcheckruntime} s) since check" + fi + if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then + touch ${FOLDER}/${nip}-${rcheck}.down + SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1 + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} + fi + echo "$(date) - TIMEOUT - ${rcheck} - $(SHOW_TIME ${replruntime} s) since copy & $(SHOW_TIME ${replcheckruntime} s) since check" >> ${FOLDER}/log.repl + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Good${idsCL[Default]} - $(SHOW_TIME ${replruntime} s) since copy & $(SHOW_TIME ${replcheckruntime} s) since check" + fi + if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then + rm -f ${FOLDER}/${nip}-${rcheck}.down + SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} + fi + echo "$(date) - SYNCED - ${rcheck} - $(SHOW_TIME ${replruntime} s) since copy & $(SHOW_TIME ${replcheckruntime} s) since check" >> ${FOLDER}/log.repl + fi + + done + else + if [ "${ST_ACTION}" != "check" ]; then + [ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}" + [ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}" + fi + fi + if [ "${ST_ACTION}" != "check" ]; then echo; fi + fi + nid=`expr $nid + 1` + done + fi + done + ########################## + # REMOVE REPL CHECK FILES + ########################## + if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}" + fi + #NHCMD="ssh root@${MYSQL_HOSTS[0]}" + #LBHCMD="ssh root@${LB_HOSTS[0]}" + #WHCMD="ssh root@${WEB_HOSTS[0]}" + for NTYPE in "${NODE_TYPES[@]}"; do + PH=${NTYPE}_HOSTS[0] + if [ ! -f ${FOLDER}/${!PH}.down ]; then + PH_CMD="ssh root@${!PH}" + var=${NTYPE}_REPL_CHECK[@] + if [ ! -z ${!var+x} ]; then + for rcheck in "${!var}"; do + ${PH_CMD} rm -f ${FOLDER}/test.repl + daterun=`date +%Y-%m-%d-%H-%M-%S` + if [ "${PH_CMD}" = "" ]; then + rm -f ${FOLDER}/test.repl & + rm -f ${REPL_CHECKS[${rcheck}]}/test.repl & + else + ${PH_CMD} rm -f ${FOLDER}/test.repl & + ${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl & + fi + done + fi + fi + done + rm -Rf ${FOLDER}/test.repl + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Complete${idsCL[Default]}" + echo + fi + fi + fi + + ######################## + ## FREE SPACE CHECK + ######################## + #if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then + if [ "${ST_ACTION}" = "freespace" ]; then + if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then + for NTYPE in "${NODE_TYPES[@]}"; do + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsST[Bold]}"; DIVIDER + echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Free Space Scan${idsCL[Default]}" + DIVIDER; echo -e "${idsST[Reset]}" + fi + nid=1 + var=${NTYPE}_HOSTS[@] + for nip in "${!var}"; do + if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' + else NCMD="ssh root@${nip}"; LH='' + fi + if [ "${ST_ACTION}" != "check" ]; then + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}" + DIVIDER false green + fi + + [ "${ST_ACTION}" != "check" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}" + declare -A partitions + DRIVEINFO=$(ssh root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }') + DRIVEINFO=(${DRIVEINFO}) + echo -en "\e[1A"; + echo -e "\e[0K\r" + + NUMDRIVES=$((${#DRIVEINFO[@]} / 3)) + for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do + ii=$((${i}*3)) + pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1 + pname=${pname#*vg-} + if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then + freespace=${DRIVEINFO[$((${ii}+2))]//M/} + if [ "${ST_ACTION}" != "check" ]; then + c=0; cw=20; spc='' + spc1=${cw}-${#pname} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${pname}$spc: " + fi + if [[ ${freespace} -gt 1024 ]]; then + fsgb=$(bc <<< "scale=2; ${freespace}/1024") + fsdsp="${fsgb} GB" + else + fsdsp="${freespace} MB" + fi + if [ "${freespace}" -le "1024" ]; then + fs_status='error' + fs_status_color='Red' + SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free" 1 + elif [ "${freespace}" -le "5120" ]; then + fs_status='warn' + fs_status_color='Yellow' + SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free" + else + fs_status='' + fs_status_color='Green' + fi + [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" + fi + done + + # for partition in "${partitions3[@]}"; do + # if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then + # pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1 + # pname=${pname#*vg-} + # if [ "${ST_ACTION}" != "check" ]; then + # c=0; cw=20; spc='' + # spc1=${cw}-${#pname} + # until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + # echo -en " ${pname}$spc: " + # fi + # + # # [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition} + # # freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1 + # + # if [ "${freespace}" -le "1000" ]; then + # fs_status='error' + # fs_status_color='Red' + # SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" 1 + # + # elif [ "${freespace}" -le "5000" ]; then + # fs_status='warn' + # fs_status_color='Yellow' + # SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" + # + # else + # fs_status='' + # fs_status_color='Green' + # + # fi + # if [ "${ST_ACTION}" != "check" ]; then + # if [[ ${freespace} -gt 1000 ]]; then + # fsgb=$(bc <<< "scale=2; ${freespace}/1000") + # fsdsp="${fsgb} GB" + # else + # fsdsp="${freespace} MB" + # fi + # echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" + # + # fi + # fi + # done + + nid=`expr $nid + 1` + echo + done + done + fi + fi + ######################## + ######################## + + if [ "${ST_ACTION}" != "check" ]; then + echo "" + if [ -z $action ] || [ "${action}" = "gui" ]; then + DIVIDER true + ENTER2CONTINUE + fi + fi + end=`date +%s` + runtime=$((end-start)) + echo "runtime: ${runtime}" +} \ No newline at end of file diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 9a24028b..2269543b 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -8,691 +8,9 @@ source /opt/idssys/defaults/default.inc source /opt/idssys/nodemgmt/settings.conf source /opt/idssys/nodemgmt/defaults.inc -#source /opt/idssys/nodemgmt/ssl-check.sh - - -# ========================================================= # - -STATUS(){ - start=`date +%s` - ST_ACTION=${1} - if [ "${ST_ACTION}" = "report" ]; then - unset idsCL idsBG idsST - idsCL=('') - idsBG=('') - idsST=('') - elif [ "${ST_ACTION}" = "sync" ]; then - ST_ACTION=repl - fi - declare -i cw; declare -i spc1; declare -i c - - ######################## - if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}" - fi - for NTYPE in "${NODE_TYPES[@]}"; do - PH=${NTYPE}_HOSTS[0] - if [ ! -f ${FOLDER}/${!PH}.down ]; then - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - for rcheck in "${!var}"; do - # echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}" - ${PH_CMD} rm -f ${FOLDER}/test.repl - daterun=`date +%Y-%m-%d-%H-%M-%S` - ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" & - done - fi - fi - done - replstart=`date +%s` - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Complete${idsCL[Default]}" - echo - fi - fi - - ######################## - if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then - if [ ! -z ${LOCAL_SERVICES+x} ]; then - if [ "${ST_ACTION}" != "check" ]; then - lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) - uptime=`uptime -p` - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" - DIVIDER false green - fi - for srvc in "${LOCAL_SERVICES[@]}"; do - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=30; spc='' - spc1=${cw}-${#NM_SERVICES[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${NM_SERVICES[${srvc}]}${spc}: " - fi - - if [ "$(systemctl is-active ${srvc})" != "active" ]; then - if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - fi - SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1 - touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} - else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then - mv ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime - fi - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down - fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then - if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) - else - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down) - fi - SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.* - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE} - - fi - fi - done - - if [ "${ST_ACTION}" != "check" ]; then echo; fi - fi - ######################## - - for NTYPE in "${NODE_TYPES[@]}"; do - nid=1 - - dockers=${NTYPE}_DOCKER[@] - hosts=${NTYPE}_HOSTS[@] - - var=${NTYPE}_HOSTS[@] - - if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" == "dockers" ]; then - GOFORCHECK=false; - else - GOFORCHECK=true; - fi - - if [ ${GOFORCHECK} = true ]; then - - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - if [ "${ST_ACTION}" = "dockers" ]; then - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" - fi - DIVIDER; echo -e "${idsST[Reset]}" - fi - for nip in "${!var}"; do - [ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}" - - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh root@${nip}"; LH='' - fi - if [ "${NCMD}" != "" ]; then - checkhost=$(CHECK_HOST ${nip}) - fi - if [ "${checkhost}" != "false" ]; then - if [ "${ST_ACTION}" != "check" ]; then - #echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}" - if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then - uptime=`${NCMD} uptime -p` - echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" - else - echo -e "${idsCL[Default]}" - fi - - DIVIDER false green - fi - if [ -f ${FOLDER}/${nip}.down ]; then - if [ -f ${FOLDER}/${nip}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) - else - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) - fi - rm -f ${FOLDER}/${nip}.* - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - fi - - if [ "${ST_ACTION}" != "dockers" ]; then - srvcs=${NTYPE}_SERVICES_CHECK[@]; - srvcstotest="$(join_by " " ${!srvcs})" - srvctst=(`${NCMD} systemctl is-active ${srvcstotest}`) - sr=0 - for srvc in "${!srvcs}"; do - [ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false || NOGOCHK=true - if [ ${NOGOCHK} == true ]; then - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=30; spc=''; spc1=${cw}-${#NM_SERVICES[${srvc}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${NM_SERVICES[${srvc}]}$spc: " - fi - if [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" == "mysql" ]; then - [ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false - elif [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" = "mariadb" ]; then - [ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false - elif [ "${srvctst[$sr]}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then - mysqlgo=true - fi - # echo "mysqlgo=${mysqlgo}" - if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 - touch ${FOLDER}/${nip}-${srvc}.down - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} - else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ] && mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - touch ${FOLDER}/${nip}-${srvc}.down - # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & - fi - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - else - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" - if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then - [ -f ${FOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} - - rm -f ${FOLDER}/${nip}-${srvc}.* - - fi - # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then - if [ "${srvc}" == "gitea" ]; then - rm -f ${FOLDER}/*-${srvc}.* - fi - fi - [ "${mysqlgo}" = "true" ] && unset mysqlgo - fi - sr=`expr $sr + 1` - done - - - - - - # for srvc in "${!var2}"; do - # NOGOCHK=true; - # [ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false; - # if [ ${NOGOCHK} = true ]; then - # if [ "${ST_ACTION}" != "check" ]; then - # if [ "${ST_ACTION}" != "report" ]; then - # c=0; cw=30; spc='' - # spc1=${cw}-${#NM_SERVICES[${srvc}]} - # until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - # else - # spc=' ' - # fi - # echo -en " ${NM_SERVICES[${srvc}]}$spc: " - # fi - # srvctst=$(${NCMD} systemctl is-active ${srvc}) - # if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then - # mysqlgo=false; - # [ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true; - # elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then - # mysqlgo=false; - # [ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true; - # elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then - # mysqlgo=true - # fi - # # echo "mysqlgo=${mysqlgo}" - # if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - # if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then - # if [ "${ST_ACTION}" != "check" ]; then - # echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - # fi - # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 - # touch ${FOLDER}/${nip}-${srvc}.down - # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} - # - # else - # errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - # if [ $errtime -gt ${RENOTIFY} ]; then - # if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - # mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime - # fi - # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - # touch ${FOLDER}/${nip}-${srvc}.down - # if [ "${srvc}" = "mysql" ]; then - # ${NCMD} systemctl restart ${srvc} - # fi - # fi - # if [ "${ST_ACTION}" != "check" ]; then - # echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - # fi - # fi - # else - # if [ "${ST_ACTION}" != "check" ]; then - # echo -e "${idsCL[Green]}Running${idsCL[Default]}" - # fi - # if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then - # if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - # else - # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - # fi - # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} - # - # rm -f ${FOLDER}/${nip}-${srvc}.down - # rm -f ${FOLDER}/${nip}-${srvc}.errtime - # - # fi - # # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then - # if [ "${srvc}" == "gitea" ]; then - # rm -f ${FOLDER}/*-${srvc}.down - # rm -f ${FOLDER}/*-${srvc}.errtime - # fi - # fi - # [ "${mysqlgo}" = "true" ] && unset mysqlgo - # fi - # done - fi - - if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo - echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}" - echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}" - fi - - for docker in "${!dockers}"; do - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=30; spc='' - spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}` - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${NM_DOCKERS[${docker}]}$spc: " - fi - - if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then - ${NCMD} docker rm ${docker} >/dev/null 2>&1 - fi - ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - sleep 10s - if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" - fi - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed" - rm -f ${FOLDER}/${nip}-${docker}.down - rm -f ${FOLDER}/${nip}-${docker}.errtime - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE} - - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" - - if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1 - touch ${FOLDER}/${nip}-${docker}.down - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE} - else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then - mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime - fi - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - touch ${FOLDER}/${nip}-${docker}.down - fi - fi - - fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - rm -f ${FOLDER}/${nip}-${docker}.down - rm -f ${FOLDER}/${nip}-${docker}.errtime - fi - done - fi - - else - if [ ! -f ${FOLDER}/${nip}.down ]; then - touch ${FOLDER}/${nip}.down - if [ ! -f ${FOLDER}/${nip}.errtime ]; then - touch ${FOLDER}/${nip}.errtime - fi - if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) - echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" - fi - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1 - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE} - else - if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) - echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" - fi - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) - # if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then - # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE} - # fi - fi - - fi - if [ "${ST_ACTION}" != "check" ]; then echo; fi - nid=`expr $nid + 1` - done - - fi #GOFORCHECK - - done - - fi - ######################## - ## REPLICATION CHECK - ######################## - - if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - fi - replcheckstart=`date +%s` - echo "$(date) - STARTING REPL CHECK - $(SHOW_TIME $(expr `date +%s` - $replstart) s)" >> ${FOLDER}/log.repl - for NTYPE in "${NODE_TYPES[@]}"; do - PH=${NTYPE}_HOSTS[0] - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - nid=1 - - var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - - # echo "CHECK: ${NTYPE} - ${nip}" - - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" - DIVIDER false green - fi - - # checkhost=$(CHECK_HOST ${nip}) - #if [ "${checkhost}" != "false" ]; then - if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then - var2=${NTYPE}_REPL_CHECK[@] - # timeout=`date --date='20 seconds' +%s` - for rcheck in "${!var2}"; do - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=30; spc='' - spc1=${cw}-${#REPL_DESC[${rcheck}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${REPL_DESC[${rcheck}]}${spc}: " - - timeout=`date --date='2 minutes' +%s` - - # if [ "${rcheck}" == "www" ]; then - # timeout=`date --date='2 minutes' +%s` - # else - # timeout=`date --date='20 seconds' +%s` - # fi - - else - timeout=`date --date='2 minutes' +%s` - fi - warn_timeout=`date --date='30 secs' +%s` - - checked=false - until [ "${checked}" = "" ]; do - if [ "${PH_CMD}" = "" ]; then - ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` - else - ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" - fi - if [ "`date +%s`" -gt "${timeout}" ]; then - timeout=true - break - elif [ "`date +%s`" -gt "${warn_timeout}" ]; then - echo -en " waiting for sync... " - warn_timeout=`date --date='100 minutes' +%s` - fi - done - replend=`date +%s` - replruntime=$((replend-replstart)) - replcheckruntime=$((replend-replcheckstart)) - if [ "${timeout}" = "true" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Timeout${idsCL[Default]} - $(SHOW_TIME ${replruntime} s) since copy & $(SHOW_TIME ${replcheckruntime} s) since check" - fi - if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then - touch ${FOLDER}/${nip}-${rcheck}.down - SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1 - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} - fi - echo "$(date) - TIMEOUT - ${rcheck} - $(SHOW_TIME ${replruntime} s) since copy & $(SHOW_TIME ${replcheckruntime} s) since check" >> ${FOLDER}/log.repl - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Good${idsCL[Default]} - $(SHOW_TIME ${replruntime} s) since copy & $(SHOW_TIME ${replcheckruntime} s) since check" - fi - if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then - rm -f ${FOLDER}/${nip}-${rcheck}.down - SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} - fi - echo "$(date) - SYNCED - ${rcheck} - $(SHOW_TIME ${replruntime} s) since copy & $(SHOW_TIME ${replcheckruntime} s) since check" >> ${FOLDER}/log.repl - fi - - done - else - if [ "${ST_ACTION}" != "check" ]; then - [ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}" - [ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}" - fi - fi - if [ "${ST_ACTION}" != "check" ]; then echo; fi - fi - nid=`expr $nid + 1` - done - fi - done - ########################## - # REMOVE REPL CHECK FILES - ########################## - if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}" - fi - #NHCMD="ssh root@${MYSQL_HOSTS[0]}" - #LBHCMD="ssh root@${LB_HOSTS[0]}" - #WHCMD="ssh root@${WEB_HOSTS[0]}" - for NTYPE in "${NODE_TYPES[@]}"; do - PH=${NTYPE}_HOSTS[0] - if [ ! -f ${FOLDER}/${!PH}.down ]; then - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - for rcheck in "${!var}"; do - ${PH_CMD} rm -f ${FOLDER}/test.repl - daterun=`date +%Y-%m-%d-%H-%M-%S` - if [ "${PH_CMD}" = "" ]; then - rm -f ${FOLDER}/test.repl & - rm -f ${REPL_CHECKS[${rcheck}]}/test.repl & - else - ${PH_CMD} rm -f ${FOLDER}/test.repl & - ${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl & - fi - done - fi - fi - done - rm -Rf ${FOLDER}/test.repl - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Complete${idsCL[Default]}" - echo - fi - fi - fi - - ######################## - ## FREE SPACE CHECK - ######################## - #if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then - if [ "${ST_ACTION}" = "freespace" ]; then - if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then - for NTYPE in "${NODE_TYPES[@]}"; do - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Free Space Scan${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - fi - nid=1 - var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh root@${nip}"; LH='' - fi - if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}" - DIVIDER false green - fi - - [ "${ST_ACTION}" != "check" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}" - declare -A partitions - DRIVEINFO=$(ssh root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }') - DRIVEINFO=(${DRIVEINFO}) - echo -en "\e[1A"; - echo -e "\e[0K\r" - - NUMDRIVES=$((${#DRIVEINFO[@]} / 3)) - for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do - ii=$((${i}*3)) - pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1 - pname=${pname#*vg-} - if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then - freespace=${DRIVEINFO[$((${ii}+2))]//M/} - if [ "${ST_ACTION}" != "check" ]; then - c=0; cw=20; spc='' - spc1=${cw}-${#pname} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${pname}$spc: " - fi - if [[ ${freespace} -gt 1024 ]]; then - fsgb=$(bc <<< "scale=2; ${freespace}/1024") - fsdsp="${fsgb} GB" - else - fsdsp="${freespace} MB" - fi - if [ "${freespace}" -le "1024" ]; then - fs_status='error' - fs_status_color='Red' - SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free" 1 - elif [ "${freespace}" -le "5120" ]; then - fs_status='warn' - fs_status_color='Yellow' - SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free" - else - fs_status='' - fs_status_color='Green' - fi - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" - fi - done - - # for partition in "${partitions3[@]}"; do - # if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then - # pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1 - # pname=${pname#*vg-} - # if [ "${ST_ACTION}" != "check" ]; then - # c=0; cw=20; spc='' - # spc1=${cw}-${#pname} - # until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - # echo -en " ${pname}$spc: " - # fi - # - # # [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition} - # # freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1 - # - # if [ "${freespace}" -le "1000" ]; then - # fs_status='error' - # fs_status_color='Red' - # SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" 1 - # - # elif [ "${freespace}" -le "5000" ]; then - # fs_status='warn' - # fs_status_color='Yellow' - # SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" - # - # else - # fs_status='' - # fs_status_color='Green' - # - # fi - # if [ "${ST_ACTION}" != "check" ]; then - # if [[ ${freespace} -gt 1000 ]]; then - # fsgb=$(bc <<< "scale=2; ${freespace}/1000") - # fsdsp="${fsgb} GB" - # else - # fsdsp="${freespace} MB" - # fi - # echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" - # - # fi - # fi - # done - - nid=`expr $nid + 1` - echo - done - done - fi - fi - ######################## - ######################## - - if [ "${ST_ACTION}" != "check" ]; then - echo "" - if [ -z $action ] || [ "${action}" = "gui" ]; then - DIVIDER true - ENTER2CONTINUE - fi - fi - end=`date +%s` - runtime=$((end-start)) - echo "runtime: ${runtime}" -} +source /opt/idssys/nodemgmt/inc/status.inc +source /opt/idssys/nodemgmt/inc/certs.inc +source /opt/idssys/nodemgmt/inc/sites.inc CHECK_DOCKER_SERVICES(){ @@ -783,445 +101,6 @@ CHECK_DOCKER_SERVICES(){ } -NEWCERT(){ - echo - if [ -z ${1+x} ]; then - echo -e -n "${idsCL[LightCyan]}Create certificate for what name (comma seperated for mutiple) : ${idsCL[Default]}" - read NEW_CERT - echo - else - NEW_CERT=${1} - fi - if [[ ${NEW_CERT} == *","* ]]; then - IFS=','; NEW_CERTS=(${NEW_CERT}); unset IFS - MAIN_CERT=${NEW_CERTS[0]} - else - MAIN_CERT=${NEW_CERT} - fi - if [ "${NEW_CERT}" != "" ]; then - echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}" - echo "" - - # echo -en "${idsCL[LightYellow]}Stopping other Webservers... ${idsCL[Default]}" - # for nip in "${WEB_HOSTS[@]}"; do - # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then - # # `ssh root@${nip} service nginx stop` - # # SERVICE nginx stop ${nip} - # fi - # done - # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" - # echo - - $CERT_DAEMON certonly --webroot -w /opt/lb-data/letsencrypt-acme-challenge -d ${NEW_CERT} - # $CERT_DAEMON certonly --dry-run --webroot -w /var/www/html -d ${NEW_CERT} - - chown -R root:le /opt/lb-data/letsencrypt - chmod -R 6775 /opt/lb-data/letsencrypt - - if [ -d /opt/lb-data/letsencrypt/live/${MAIN_CERT} ]; then - - touch /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert - if [ -f /opt/lb-data/nginx/sites-enabled/${MAIN_CERT}.conf ]; then - rm -f ${FOLDER}/cert-request.lastrun - daterun=`date +%Y-%m-%d-%H-%M-%S` - echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun - yes | cp -rfH ${FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun - DIVIDER true - echo -e -n "${idsCL[LightCyan]}Reload NGINX on LB Nodes (Y/n): ${idsCL[Default]}" - read -n 1 NGINXRELOAD - if [[ ${NGINXRELOAD} =~ ^[Nn]$ ]]; then - tmp='' - else - echo - echo -en "${idsCL[LightYellow]}Waiting for Cert replication across the nodes... ${idsCL[Default]}" - for nip in "${LB_HOSTS[@]}"; do - checkhost=$(CHECK_HOST ${nip}) - if [ "${checkhost}" != "false" ]; then - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then - checked=false - until [ "${checked}" = "" ]; do - checked=`ssh root@${nip} "[ ! -f /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert ] && echo '.'"` - done - fi - fi - done - rm -f /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert - echo -e "${idsCL[Green]}Completed${idsCL[Default]}" - - echo - SERVICE nginx reload - fi - fi - echo - echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}" - else - echo - echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}" - fi - - # echo - # echo -en "${idsCL[LightYellow]}Starting other Webservers... ${idsCL[Default]}" - # for nip in "${WEB_HOSTS[@]}"; do - # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then - # `ssh root@${nip} service nginx start` - # # SERVICE nginx start ${nip} - # fi - # done - # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" - - echo "" - if [ -z $action ] || [ "${action}" = "gui" ]; then - DIVIDER true - ENTER2CONTINUE - fi - else - echo "You havent entered a site address." - sleep 1 - NEWCERT - exit 0 - fi -} - -DEL-SSL(){ - if [ -z ${1+x} ]; then - echo -e -n "${idsCL[LightCyan]}Delete what SSL site address: ${idsCL[Default]}" - read DEL_SSL - echo "" - else - DEL_SSL=${1} - fi - if [ ! -z ${DEL_SSL+x} ] && [ "${DEL_SSL}" != "" ]; then - if [ -d /etc/letsencrypt/live/${DEL_SSL} ]; then - echo -e "${idsCL[LightRed]}Deleting the SSL certificates for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}'...${idsCL[Default]}" - echo "" - - echo -en "${idsCL[LightRed]}Removing Files and Folders... ${idsCL[Default]}" - rm -rf /etc/letsencrypt/archive/${DEL_SSL} - rm -rf /etc/letsencrypt/live/${DEL_SSL} - rm -f /etc/letsencrypt/renewal/${DEL_SSL}.conf - echo -e "${idsCL[Green]}OK${idsCL[Default]}" - echo "" - - - else - echo -e "${idsCL[LightRed]}The SSL files folder for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}' could not be found.${idsCL[Default]}" - exit 1 - fi - if [ "${timeout}" != "true" ]; then - echo -e "${idsCL[Green]}Completed${idsCL[Default]}" - fi - echo - # SERVICE nginx reload - echo -e "${idsCL[LightRed]}The SSL certificate has been removed fromt be nodes.${idsCL[Default]}" - fi -} - -CERTRENEW(){ - echo -en "${idsCL[LightCyan]}Stopping Webserver-Node2...${idsCL[Default]}" - ssh root@webserver-node2.scity.us service nginx stop - echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}" - echo - echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}" - echo - sleep 5 - $CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun - # $CERT_DAEMON renew --force-renewal --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun - # $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun - CONCAT_SSL - chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun - chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun - yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun - daterun=`date +%Y-%m-%d-%H-%M-%S` - echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun - DIVIDER true - echo -en "${idsCL[LightCyan]}Starting Webserver-Node2 Back up...${idsCL[Default]}" - ssh root@webserver-node2.scity.us service nginx start - echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}" - echo - SERVICE nginx reload 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun - echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}" - echo "" - if [ -z $action ] || [ "${action}" = "gui" ]; then - DIVIDER true - ENTER2CONTINUE - fi -} -NIGHTLYRENEW(){ - rm -f ${FOLDER}/cert-renewal.lastrun - ssh root@webserver-node2.scity.us service nginx stop - sleep 5 - $CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun - CONCAT_SSL - chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun - chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun - yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun - daterun=`date +%Y-%m-%d-%H-%M-%S` - echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun - ssh root@webserver-node2.scity.us service nginx start - SERVICE nginx reload web &>> ${FOLDER}/cert-renewal.lastrun -} - -CONCAT_SSL(){ - rm -f /tmp/ssllist - for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist - for certdir in $( ${certdir}fullcert.pem - done -} - -LISTCERTS(){ - declare -i cw; declare -i spc1; declare -i c - declare -A CHECKCERT_DOMAINS - IFS=$'\n' - for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do - HOST=${LINE%% *} - PORT=${LINE#* } - IFS=" " - CHECKCERT_DOMAINS[${HOST}]=${PORT} - done - unset IFS - if [ ! -z ${LOCAL_SERVICES+x} ]; then - NCMD="ssh root@${MYSQL_HOSTS[0]}" - #${NCMD} rm -f /tmp/ssllist - #${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist - else - NCMD='' - rm -f /tmp/ssllist - #for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist - fi - if [ -z $action ] || [ "${action}" = "gui" ]; then - DIVIDER true - fi - echo - echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}" - DIVIDER false yellow 120 - echo -e "Subject Name Monitored Expires Alternate Subject Names" - DIVIDER false yellow 120 - - cw=30; - c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done - - #ssldir=$(${NCMD} find /opt/nginx-proxy/ssl/* -type l) - ssldir=$(${NCMD} find /opt/lb-data/letsencrypt/live/* -type d) - for certdir in ${ssldir[@]}; do - SUBJECT=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV) - SUBJECTNAMES=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV) - CERTEXPIRE=$(date -d "$(: | ${NCMD} openssl x509 -in ${certdir}/cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s'); - - SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, } - # SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/\n/, /g") - SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g") - SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g") - SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g") - IFS=', '; SUBJECT_NAMES=(${SUBJECTNAMES}); unset IFS - DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS))); - - c=0; spc='' - spc1=${cw}-${#SUBJECT} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - if [ $CERTEXPIRE -le $DUEIN ]; then - date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}" - SENDNOTICE "SSL Expiring" "${SUBJECT} expires on ${date}" 1 - else - date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}" - fi - - if [ "${CHECKCERT_DOMAINS[${SUBJECT}]}" = "" ]; then - monitored='No ' - else - monitored="${idsCL[Green]}Yes${idsCL[Default]}" - fi - if [ ${#SUBJECT_NAMES[@]} -lt 4 ]; then - echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored} ${date} ${SUBJECTNAMES}" - elif [ ${#SUBJECT_NAMES[@]} -lt 7 ]; then - echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored} ${date} ${SUBJECT_NAMES[0]}, ${SUBJECT_NAMES[1]}, ${SUBJECT_NAMES[2]}" - echo -e "${spc2} ${SUBJECT_NAMES[3]}, ${SUBJECT_NAMES[4]}, ${SUBJECT_NAMES[5]}" - - fi - DIVIDER false darkGray 120 - done - echo - if [ -z $action ] || [ "${action}" = "gui" ]; then - DIVIDER true - ENTER2CONTINUE - fi - echo "" -} - -LISTCERTS_NPM(){ - declare -i cw; declare -i spc1; declare -i c - declare -A CHECKCERT_DOMAINS - IFS=$'\n' - for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do - HOST=${LINE%% *} - PORT=${LINE#* } - IFS=" " - CHECKCERT_DOMAINS[${HOST}]=${PORT} - done - unset IFS - if [ ! -z ${LOCAL_SERVICES+x} ]; then - NCMD="ssh root@${MYSQL_HOSTS[0]}" - ${NCMD} rm -f /tmp/ssllist - ${NCMD} 'for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist - else - NCMD='' - rm -f /tmp/ssllist - for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist - fi - if [ -z $action ] || [ "${action}" = "gui" ]; then - DIVIDER true - fi - echo - echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}" - DIVIDER false yellow 120 - echo -e "Subject Name Monitored Expires Alternate Subject Names" - DIVIDER false yellow 120 - - cw=30; - c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done - for certdir in $( ${FOLDER}/ssl-cert-check.lastrun - SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)" - SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG" - else - # ${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains - IFS=$'\n' - for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do - HOST=${LINE%% *} - PORT=${LINE#* } - CERTINFO=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p ${PORT} -s ${HOST} -N) - CERTVALID=$(echo ${CERTINFO} | awk -F' valid' '{print $1}') - CERTVALID=${CERTVALID: -1} - CERTDAYS=${CERTINFO#*days=} - if [ "${CERTVALID}" == "0" ] || [ "${CERTVALID}" == "E" ]; then - printf "%-30s: %s\n" "${HOST}" "SSL is not valid" - SENDNOTICE "SSL Not Valid" "The (${HOST}) SSL is not valid" 1 - - elif [ ${CERTDAYS} -lt 1 ]; then - printf "%-30s: %s\n" "${HOST}" "SSL Expired! (${CERTDAYS} days)" - SENDNOTICE "SSL Expired" "The (${HOST}) SSL is expired!! (${CERTDAYS} days)" 2 - - elif [ ${CERTDAYS} -lt 15 ]; then - printf "%-30s: %s\n" "${HOST}" "SSL is expiring in $CERTDAYS days!" - SENDNOTICE "SSL Expiring" "The (${HOST}) SSL is expiring in ${CERTDAYS} days" 1 - - else - printf "%-30s: %s\n" "${HOST}" "SSL is valid for ${CERTDAYS} days" - fi - done - unset IFS - fi -} - - - BACKUP(){ BACKUP_FOLDER=/opt/idssys/backups/node-backup #BACKUP_FOLDER=/opt/idssys/backups/node-backups/${NODE_HOSTNAME} #/`date +%Y-%m-%d` @@ -1254,443 +133,7 @@ BACKUP(){ fi } -DELSITE(){ - while [ $# -gt 0 ]; do - case "$1" in - -site) DEL_SITE=${2};; - -ssl) DEL_SSL=${2};; - -list) DELSITES; exit 0;; - -*) - echo "Invalid option: '${1}' requires an argument" 1>&2 - echo "" - echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {" - width=33 - printf "%-${width}s- %s\n" " -site {FQDN address}" "(*required)" - printf "%-${width}s- %s\n" " -ssl {yes or [no]}" "Delete SSL certs as well" - printf "%-${width}s- %s\n" " -list" "List sites (same as running nodemgmt delsites)" - echo "}" - exit 1;; - esac - shift - done - if [ -z ${DEL_SITE+x} ]; then - echo -e -n "${idsCL[LightCyan]}Delete what site address: ${idsCL[Default]}" - read DEL_SITE - echo "" - fi - if [[ $DEL_SSL =~ ^[Nn]$ ]]; then - DEL_SSL=no - elif [[ $DEL_SSL =~ ^[Yy]$ ]]; then - DEL_SSL=yes - elif [ -z ${DEL_SSL+x} ]; then - echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${DEL_SITE}' as well? [y/N]${idsCL[Default]} " - read DEL_SSL - fi - if [ ! -z ${DEL_SITE+x} ] && [ "${DEL_SITE}" != "" ]; then - echo -e "${idsCL[LightRed]}Deleting site '${idsCL[Red]}${DEL_SITE^^}${idsCL[LightRed]}'...${idsCL[Default]}" - echo "" - - echo -e "${idsCL[LightRed]}[[Removing Files and Folders]]${idsCL[Default]}" - echo -e "${idsCL[LightRed]}-------------------------------------------${idsCL[Default]}" - echo - echo -en "${idsCL[LightCyan]}Removing files from all Nodes ... ${idsCL[Default]}" - ssh root@10.10.1.120 rm -f /etc/nginx/sites-enabled/${DEL_SITE}* >/dev/null 2>&1 - ssh root@10.10.10.80 rm -f /etc/nginx/sites-enabled/${DEL_SITE}* >/dev/null 2>&1 - if [ "${DEL_SSL}" == "yes" ]; then - ssh root@10.10.10.80 rm -rf /etc/letsencrypt/archive/${DEL_SITE} >/dev/null 2>&1 - ssh root@10.10.10.80 rm -rf /etc/letsencrypt/live/${DEL_SITE} >/dev/null 2>&1 - ssh root@10.10.10.80 rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf >/dev/null 2>&1 - fi - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - echo - echo -en "${idsCL[LightCyan]}Reloading NGINX ... ${idsCL[Default]}" - ssh root@10.10.10.80 nodemgmt service nginx reload >/dev/null 2>&1 - ssh root@10.10.1.120 nodemgmt service nginx reload >/dev/null 2>&1 - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - - # nid=1 - # for nip in "${WEB_HOSTS[@]}"; do - # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then - # nip='localhost ' - # NCMD='' - # else - # NCMD="ssh root@${nip}" - # fi - # echo -en "Removing from Webserver-Node${nid} ($nip)... ${idsCL[Default]}" - # if [ "${NCMD}" != "" ]; then - # checkhost=$(CHECK_HOST ${nip}) - # fi - # if [ "${checkhost}" != "false" ]; then - # #if [ -f /etc/nginx/sites-available/${DEL_SITE} ]; then - # ${NCMD} rm -f /etc/nginx/sites-available/${DEL_SITE}* - # ${NCMD} rm -f /etc/nginx/sites-enabled/${DEL_SITE}* - # #fi - # #if [ -d /var/www/${DEL_SITE} ]; then - # ${NCMD} rm -rf /var/www/${DEL_SITE} - # #fi - # if [ "${DEL_SSL}" = "yes" ]; then - # ${NCMD} rm -rf /etc/letsencrypt/archive/${DEL_SITE} - # ${NCMD} rm -rf /etc/letsencrypt/live/${DEL_SITE} - # ${NCMD} rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf - # fi - # echo -e "${idsCL[Green]}OK${idsCL[Default]}" - # else - # echo -e "${idsCL[Red]}Node is down${idsCL[Default]}" - # fi - # nid=`expr $nid + 1` - # done - # echo "" - # SERVICE nginx reload ns - echo -e "${idsCL[LightRed]}Site has been deleted.${idsCL[Default]}" - else - echo "Missing arguments" - echo "" - echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {" - width=33 - printf "%-${width}s- %s\n" " -site {FQDN address}" "Site to delete" - printf "%-${width}s- %s\n" " -ssl {yes or [no]}" "Delete SSL certs as well" - printf "%-${width}s- %s\n" " -list" "List sites (same as running nodemgmt delsites)" - echo "}" - exit 1 - fi -} -DELSITES(){ - echo - echo -e "${idsCL[Red]}Select a site to delete...${idsCL[Default]}" - DIVIDER true - sid=1 - filels="( $(ssh root@${WEB_HOSTS[0]} ls '/etc/nginx/sites-available/*') )" - # IFS='\n' - for siteconf in $filels; do - # for siteconf in /etc/nginx/sites-available/* ; do - # [ -e "$siteconf" ] || continue - if [ ${siteconf:0:1} == '/' ]; then - IFS='/'; site_conf=(${siteconf}); unset IFS - SITES[${sid}]=${site_conf[4]} - sid=`expr $sid + 1` - fi - done - for s in "${!SITES[@]}"; do - echo " [${s}] ${SITES[${s}]}" - done - echo "" - if [ -z $action ] || [ "${action}" = "gui" ]; then - echo " [B] Back" - fi - echo " [Q] Quit" - echo "" - echo -e -n "${idsCL[LightYellow]}Please select a site from above from above:${idsCL[Default]} " - read selsite - echo "" - if [ -z ${SITES[$selsite]} ] && [ "${selsite}" != "Q" ] && [ "${selsite}" != "q" ] && [ "${selsite}" != "B" ] && [ "${selsite}" != "b" ]; then - echo "Thats an invaild option," - echo "please select a valid option only." - sleep 1 - DELSITES - exit 0 - elif [ "${selsite}" = "Q" ] || [ "${selsite}" = "q" ]; then - exit 0 - elif [ "${selsite}" = "B" ] || [ "${selsite}" = "b" ]; then - GUI - else - while : - do - echo -e -n "${idsCL[LightRed]}Are you sure you want to delete '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}'? [y/N]${idsCL[Default]} " - read response - echo "" - if [[ $response =~ ^[Yy]$ ]]; then - echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}', if they exist? [y/N]${idsCL[Default]} " - read sslresponse - DELSITE -site ${SITES[${selsite}]} -ssl ${sslresponse} - echo "" - DIVIDER - ENTER2CONTINUE - break - else - break - fi - done - DELSITES - exit 0 - fi - if [ -z $action ] || [ "${action}" = "gui" ]; then - ENTER2CONTINUE - fi -} - -NEWSITE(){ - echo - while [ $# -gt 0 ]; do - case "$1" in - -site) NEW_SITE=${2};; - -type) SITE_TYPE=${2};; - -ssl) CREATE_SSL=${2};; - -proxy_scheme) PROXYSCHEME=${2};; - -proxy_host) PROXYHOST=${2};; - -proxy_port) PROXYPORT=${2};; - -websocket) WEBSOCKET=${2};; - -hsts) HSTS=${2};; - -exploits) EXPLOITS=${2};; - -secure) SECURE=${2};; - -h | -help | --help) - echo "" - echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {" - width=33 - printf "%-${width}s- %s\n" " -site {FQDN address(,es)}" "(new site and aliases, comma separated)" - printf "%-${width}s- %s\n" " -ssl {yes or no}" "(defaults to yes)" - printf "%-${width}s- %s\n" " -type {'local' or 'proxy'}" "(defaults to local)" - printf "%-${width}s- %s\n" " -proxy_port {host port}" "(proxy backend host)" - printf "%-${width}s- %s\n" " -proxy_host {IP or FQDN}" "(proxy backend port)" - printf "%-${width}s- %s\n" " -proxy_scheme {http or https}" "(proxy backend scheme)" - printf "%-${width}s- %s\n" " -websocket {yes or no}" "(websocket support)" - printf "%-${width}s- %s\n" " -hsts {yes or no}" "(hsts support)" - printf "%-${width}s- %s\n" " -exploits {yes or no}" "(block exploits)" - printf "%-${width}s- %s\n" " -secure {yes or no}" "(secure access [nginx/.htpasswd])" - echo "}" - exit 1;; - esac - shift - done - - #if [ -z ${SITE_TYPE+x} ]; then SITE_TYPE=local; fi - #if [ -z ${CREATE_SSL+x} ]; then CREATE_SSL=true; fi - if [ -z ${NEW_SITE+x} ]; then - echo -e -n "${idsCL[LightCyan]}New site domain name (comma seperated for multiple) : ${idsCL[Default]}" - read NEW_SITE - showdivide=yes - fi - - if [ -z ${CREATE_SSL+x} ]; then - echo -e -n "${idsCL[LightCyan]}Create SSL for site? [Y/n] ${idsCL[Default]}" - read CREATE_SSL - showdivide=yes - if [[ $CREATE_SSL =~ ^[Yy]$ ]] || [ "${CREATE_SSL}" = "" ]; then - CREATE_SSL=yes - # echo -e -n "${idsCL[LightCyan]}Add additonal domain names to the SSL cert (comma seperated)? : ${idsCL[Default]}" - # read ssladd - else - CREATE_SSL=no - fi - fi - - if [ -z ${SITE_TYPE+x} ]; then - echo -e -n "${idsCL[LightCyan]}Site type (local/{proxy}): ${idsCL[Default]}" - read SITE_TYPE - showdivide=yes - if [ "${SITE_TYPE}" = "" ]; then - SITE_TYPE=proxy - fi - fi - if [ "${SITE_TYPE}" = "proxy" ]; then - if [ -z ${PROXYHOST+x} ]; then - echo -e -n "${idsCL[LightCyan]}What is the proxy backend address (IP or FQDN): ${idsCL[Default]}" - read PROXYHOST - showdivide=yes - fi - if [ -z ${PROXYPORT+x} ]; then - echo -e -n "${idsCL[LightCyan]}What is the proxy backend port (tcp port): ${idsCL[Default]}" - read PROXYPORT - showdivide=yes - fi - if [ -z ${PROXYSCHEME+x} ]; then - echo -e -n "${idsCL[LightCyan]}What is the proxy backend scheme (http/https): ${idsCL[Default]}" - read PROXYSCHEME - showdivide=yes - fi - if [ -z ${WEBSOCKET+x} ]; then - echo -e -n "${idsCL[LightCyan]}Enable Websocket Support (y/N): ${idsCL[Default]}" - read WEBSOCKET - showdivide=yes - if [[ ${WEBSOCKET} =~ ^[Nn]$ ]] || [ "${WEBSOCKET}" = "" ]; then - WEBSOCKET=no - elif [[ ${WEBSOCKET} =~ ^[Yy]$ ]]; then - WEBSOCKET=yes - else - WEBSOCKET=no - fi - fi - if [ -z ${HSTS+x} ]; then - echo -e -n "${idsCL[LightCyan]}Enable HSTS Support (Y/n): ${idsCL[Default]}" - read HSTS - showdivide=yes - [[ ${HSTS} =~ ^[Yy]$ ]] || [ "${HSTS}" = "" ] &&HSTS=yes || HSTS=no - fi - if [ -z ${EXPLOITS+x} ]; then - echo -e -n "${idsCL[LightCyan]}Block exploits (y/N): ${idsCL[Default]}" - read EXPLOITS - showdivide=yes - if [[ ${EXPLOITS} =~ ^[Nn]$ ]] || [ "${EXPLOITS}" = "" ]; then - EXPLOITS=no - elif [[ ${EXPLOITS} =~ ^[Yy]$ ]]; then - EXPLOITS=yes - else - EXPLOITS=no - fi - fi - if [ -z ${SECURE+x} ]; then - echo -e -n "${idsCL[LightCyan]}Secure site with passwords from [ nginx/.htpasswd ](y/N): ${idsCL[Default]}" - read SECURE - showdivide=yes - if [[ ${SECURE} =~ ^[Nn]$ ]] || [ "${SECURE}" = "" ]; then - SECURE=no - elif [[ ${SECURE} =~ ^[Yy]$ ]]; then - SECURE=yes - else - SECURE=no - fi - fi - fi - [ "${showdivide}" == "yes" ] && DIVIDER - echo - width=14 - printf "%-${width}s: %s\n" "New site" "${NEW_SITE}" - printf "%-${width}s: %s\n" "Create SSL" "${CREATE_SSL}" - printf "%-${width}s: %s\n" "Site type" "${SITE_TYPE}" - if [ "${SITE_TYPE}" = "proxy" ]; then - printf "%-${width}s: %s\n" "Proxy host" "${PROXYHOST}" - printf "%-${width}s: %s\n" "Proxy port" "${PROXYPORT}" - printf "%-${width}s: %s\n" "Proxy scheme" "${PROXYSCHEME}" - printf "%-${width}s: %s\n" "Websocket Support" "${WEBSOCKET}" - printf "%-${width}s: %s\n" "HSTS Support" "${HSTS}" - printf "%-${width}s: %s\n" "Block Exploits" "${EXPLOITS}" - printf "%-${width}s: %s\n" "Secure Access" "${SECURE}" - fi - echo -e -n "${idsCL[LightRed]}Is this information correct? [Y/n]${idsCL[Default]} " - read -n 1 response - echo - if [[ $response =~ ^[Yy]$ ]] || [ "${response}" = "" ]; then - if [[ ${NEW_SITE} == *","* ]]; then - IFS=','; NEW_SITES=(${NEW_SITE}); unset IFS - MAIN_SITE=${NEW_SITES[0]} - NGINX_SERVERNAME=${NEW_SITE//[,]/ } - else - MAIN_SITE=${NEW_SITE} - NGINX_SERVERNAME=${NEW_SITE} - - fi - if [ "${SITE_TYPE}" = "proxy" ]; then - if [ ! -z ${PROXYSCHEME+x} ] && [ ! -z ${PROXYHOST+x} ] && [ ! -z ${PROXYPORT+x} ]; then GO=true; fi - else GO=true - fi - if [ "${GO}" = "true" ]; then - echo -e "${idsCL[LightGreen]}Setting up new site for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}" - echo "" - - [ "${WEBSOCKET}" == "yes" ] && WEBSOCKET="include conf.d\/include\/websocket-support.conf;" || WEBSOCKET="" - [ "${HSTS}" == "yes" ] && HSTS="include conf.d\/include\/hsts-support.conf;" || HSTS="" - [ "${EXPLOITS}" == "yes" ] && EXPLOITS="include conf.d\/include\/block-exploits.conf;" || EXPLOITS="" - [ "${SECURE}" == "yes" ] && SECURE="include conf.d\/include\/secure-access.conf;" || SECURE="" - - - ######################################### LOCAL - if [ "${SITE_TYPE}" = "local" ]; then - echo -e "server { - listen 80;" > /etc/nginx/sites-available/${MAIN_SITE} - if [ "${CREATE_SSL}" = "yes" ]; then - echo -e " listen 443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE} - fi - echo -e " - server_name ${NGINX_SERVERNAME}; - - set \$base /var/www/${MAIN_SITE}; - root \$base/public_html; - - access_log /var/log/nginx/${MAIN_SITE}-access.log; - error_log /var/log/nginx/${MAIN_SITE}-error.log warn;" >> /etc/nginx/sites-available/${MAIN_SITE} - - if [ "${CREATE_SSL}" = "yes" ]; then - echo -e " - ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem; - include conf.d/include/ssl-ciphers.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} - fi - echo -e " - index index.php; - - location / { - try_files \$uri \$uri/ /index.php?\$query_string;" >> /etc/nginx/sites-available/${MAIN_SITE} - if [ "${CREATE_SSL}" = "yes" ]; then - echo -e " include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} - fi - echo -e " } - - location ~ \.php\$ { - fastcgi_pass unix:/var/run/php/php7.2-fpm.sock; - include conf.d/include/php_fastcgi.conf; - } - - include conf.d/include/general.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} - if [ "${CREATE_SSL}" = "yes" ]; then - echo -e " include conf.d/include/letsencrypt-acme-challenge.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} - fi - echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE} - - - sudo -u www-data mkdir -p /var/www/${MAIN_SITE}/{public_html,nginx_logs} - # echo -en "${idsCL[LightYellow]}Waiting for folder replication across the webserver nodes... ${idsCL[Default]}" - # for nip in "${WEB_HOSTS[@]}"; do - # checkhost=$(CHECK_HOST ${nip}) - # if [ "${checkhost}" != "false" ]; then - # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then - # checked=false - # until [ "${checked}" = "" ]; do - # checked=`ssh root@${nip} "[ ! -d /var/www/${MAIN_SITE} ] && echo does not exist"` - # done - # fi - # fi - # done - # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" - # - # echo -en "${idsCL[LightYellow]}Setting folder permissions... ${idsCL[Default]}" - # SET-PERMISSIONS ${MAIN_SITE} - # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" - - - - ######################################### PROXY - else - - cp ${FOLDER}/templates/nginx.proxy.site /etc/nginx/sites-enabled/${MAIN_SITE}.conf - sed -i "s/<>/${NGINX_SERVERNAME}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - sed -i "s/<>/${MAIN_SITE}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - sed -i "s/<>/${PROXYHOST}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - sed -i "s/<>/${PROXYPORT}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - sed -i "s/<>/${PROXYSCHEME}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - sed -i "s/<>/${WEBSOCKET}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - sed -i "s/<>/${HSTS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - sed -i "s/<>/${EXPLOITS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - sed -i "s/<>/${SECURE}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - fi - - if [ "${CREATE_SSL}" = "yes" ]; then - [ "${ssladd}" != "" ] && NEWCERT ${NEW_SITE},${ssladd} || NEWCERT ${NEW_SITE} - if [ "${SITE_TYPE}" == "proxy" ]; then - sed -i "s/#ssl_certificate/ssl_certificate/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf - fi - fi - - rm -f ${FOLDER}/new-site.lastrun - daterun=`date +%Y-%m-%d-%H-%M-%S` - echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun - # yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun - # yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun - # daterun=`date +%Y-%m-%d-%H-%M-%S` - # echo -e "${daterun}" >> /etc/nginx/new-site.lastrun - DIVIDER true - echo "" - echo -e "${idsCL[LightGreen]}The new site for '${idsCL[LightGreen]}${NEW_SITE}${idsCL[Default]}' has been created.${idsCL[Default]}" - echo "" - else - echo "Missing proxy arguments" - Exit 1 - fi - - else - ${SCRIPT} newsite - exit 0 - fi -} SET-PERMISSIONS(){ if [ "${1}" = "gitea" ]; then