#!/usr/bin/env bash # NodeMgmt - Galera/NGINX Node Management Scripts action="$1" if [ ! -f /opt/idssys/settings/nodemgmt.conf ]; then if [ ! -d /opt/idssys/settings ]; then mkdir /opt/idssys/settings fi mv /opt/idssys/nodemgmt/settings.conf /opt/idssys/settings/nodemgmt.conf fi source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc source /opt/idssys/settings/nodemgmt.conf source /opt/idssys/nodemgmt/defaults.inc # ========================================================= # STATUS() { if [ ! -z ${LOCAL_SERVICES+x} ]; then lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) #DIVIDER true echo -e "Node hostname: ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - localhost${idsCL[Default]}" for srvc in "${LOCAL_SERVICES[@]}" do if [[ "mysql,nginx,gitea,haproxy,keepalived,maxscale" = *"${srvc}"* ]]; then spc=" "; else spc=""; fi echo -en "${NM_SERVICES[${srvc}]} $spc" if [ $(pgrep ${srvc} | wc -l) -gt "0" ]; then echo -e "${idsCL[Green]}Running${idsCL[Default]}" else echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi done fi nid=1 for nip in "${NODE_HOSTS[@]}" do if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' else NCMD="ssh root@${nip}"; LH='' fi DIVIDER true echo -e "Node hostname: ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${LH}${idsCL[Default]}" for srvc in "${NODE_SERVICES[@]}" do if [[ "mysql,nginx,gitea,haproxy,keepalived,maxscale" = *"${srvc}"* ]]; then spc=" "; else spc=""; fi echo -en "${NM_SERVICES[${srvc}]} $spc" if [ $(${NCMD} pgrep ${srvc} | wc -l) -gt "0" ]; then echo -e "${idsCL[Green]}Running${idsCL[Default]}" else echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi done nid=`expr $nid + 1` done DIVIDER echo "" if [ -z $action ] || [ "${action}" = "gui" ]; then ENTER2CONTINUE fi } NEWCERT(){ echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${1}${idsCL[LightGreen]}'...${idsCL[Default]}" echo "" do_with_root $CERT_DAEMON certonly --webroot -w /var/www/html -d ${1} do_with_root chown -R root:letsencrypt /etc/letsencrypt do_with_root chmod -R 6775 /etc/letsencrypt echo -e "${idsCL[LightYellow]}Waiting for certifcate replication between the nodes...${idsCL[Default]}" echo "" sleep 20 SERVICE nginx reload } CERTRENEW(){ echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}" do_with_root $CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee /opt/idssys/nodemgmt/cert-renewal.lastrun do_with_root chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a /opt/idssys/nodemgmt/cert-renewal.lastrun do_with_root chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a /opt/idssys/nodemgmt/cert-renewal.lastrun echo -e "${idsCL${idsCL[LightYellow]}}Waiting for certifcate replication between the nodes...${idsCL[Default]}" echo "" sleep 20 SERVICE nginx reload 2>&1 | tee -a /opt/idssys/nodemgmt/cert-renewal.lastrun } LISTCERTS(){ #DIVIDER true for certdir in /etc/letsencrypt/temp/*/ ; do SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV) SUBJECTNAMES=$(openssl x509 -in ${certdir}/cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV) SUBJECTNAMES=${SUBJECTNAMES//$'\n'/,} SUBJECTNAMES2=$(echo $SUBJECTNAMES | sed "s/${SUBJECT},//g") SUBJECTNAMES2=$(echo $SUBJECTNAMES2 | sed "s/,${SUBJECT}//g") echo $SUBJECTNAMES2 IFS=','; SUBJECT_NAMES=(${SUBJECTNAMES}); unset IFS CERTEXPIRE=$(date -d "$(: | openssl x509 -in ${certdir}/cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s'); DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS))); DIVIDER true echo -e "Certificate Subject: ${idsST[Bold]}${idsBG[Blue]}${SUBJECT}${idsST[Reset]}${idsBG[Default]}" if [ "${#SUBJECT_NAMES[*]}" -gt 1 ]; then anid=1 for ALTNAME in "${SUBJECT_NAMES[@]}"; do if [ "${anid}" = 1 ]; then alttitle='Alternate Subject(s):'; else alttitle=' '; fi echo -e "${alttitle} ${idsCL[LightCyan]}${ALTNAME}${idsCL[Default]}" anid=`expr $anid + 1` done fi if [ $DUEIN -gt $CERTEXPIRE ]; then echo -e "Expiration Date: ${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}" else echo -e "Expiration Date: ${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}" fi echo "" if [ "${#SUBJECT_NAMES[*]}" -gt 1 ]; then echo "-d ${SUBJECT},${SUBJECTNAMES2}" else echo "-d ${SUBJECT}" fi echo "" done if [ -z $action ] || [ "${action}" = "gui" ]; then ENTER2CONTINUE fi echo "" } NIGHTLYRENEW(){ rm -f /opt/idssys/nodemgmt/cert-renewal.lastrun do_with_root $CERT_DAEMON renew --webroot -w /var/www/html &>> /opt/idssys/nodemgmt/cert-renewal.lastrun do_with_root chown -R root:letsencrypt /etc/letsencrypt &>> /opt/idssys/nodemgmt/cert-renewal.lastrun do_with_root chmod -R 6775 /etc/letsencrypt &>> /opt/idssys/nodemgmt/cert-renewal.lastrun sleep 20 SERVICE nginx reload &>> /opt/idssys/nodemgmt/cert-renewal.lastrun } DELSITE(){ if [ ! -z ${1+x} ]; then DEL_SITE=${1} echo -e "${idsCL[LightRed]}Deleting site '${idsCL[Red]}${DEL_SITE}${idsCL[LightRed]}'...${idsCL[Default]}" echo "" echo -e "${idsCL[LightRed]}[[Removing Files and Folders]]${idsCL[Default]}" echo -e "${idsCL[LightRed]}-------------------------------------------${idsCL[Default]}" nid=1 for nip in "${NODE_HOSTS[@]}" do if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then nip='localhost ' NCMD='' else NCMD="ssh root@${nip}" fi echo -en "Removing from iDS-Node${nid} ($nip)... ${idsCL[Default]}" if [ -f /etc/nginx/sites-available/${DEL_SITE}.conf ]; then ${NCMD} rm -f /etc/nginx/sites-available/${DEL_SITE}.conf ${NCMD} rm -f /etc/nginx/sites-enabled/${DEL_SITE}.conf fi if [ -d /var/www/${DEL_SITE} ]; then ${NCMD} rm -rf /var/www/${DEL_SITE} fi if [ -d /etc/letsencrypt/archive/${DEL_SITE} ]; then ${NCMD} rm -rf /etc/letsencrypt/archive/${DEL_SITE} ${NCMD} rm -rf /etc/letsencrypt/live/${DEL_SITE} ${NCMD} rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf fi echo -e "${idsCL[Green]}OK${idsCL[Default]}" nid=`expr $nid + 1` done echo "" SERVICE nginx reload echo -e "${idsCL[LightRed]}Site has been deleted.${idsCL[Default]}" else echo -e "${idsCL[LightRed]}No site was defined.${idsCL[Default]}" fi } NEWSITE(){ while [ $# -gt 0 ]; do case "$1" in -site) NEW_SITE=${2};; -type) SITE_TYPE=${2};; -ssl) CREATE_SSL=${2};; -proxy_scheme) PROXYSCHEME=${2};; -proxy_host) PROXYHOST=${2};; -proxy_port) PROXYPORT=${2};; -*) echo "Invalid option: '${1}' requires an argument" 1>&2 echo "" echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {" width=33 printf "%-${width}s- %s\n" " -site {FQDN address}" "(*required)" printf "%-${width}s- %s\n" " -ssl {[true] or false}" "" printf "%-${width}s- %s\n" " -type {[local] or proxy}" "" printf "%-${width}s- %s\n" " -scheme {http or https}" "(required if type set to proxy)" printf "%-${width}s- %s\n" " -host {IP or FQDN}" "(required if type set to proxy)" printf "%-${width}s- %s\n" " -port {host port}" "(required if type set to proxy)" echo "}" exit 1;; esac shift done if [ -z ${SITE_TYPE+x} ]; then SITE_TYPE=local; fi if [ -z ${CREATE_SSL+x} ]; then CREATE_SSL=true; fi if [ "${NEW_SITE}" != "" ]; then if [[ ${NEW_SITE} == *","* ]]; then IFS=, NEW_SITES=(${NEW_SITE}) unset IFS MAIN_SITE=${NEW_SITES[0]} NGINX_SERVERNAME=${NEW_SITE//[,]/ } else MAIN_SITE=${NEW_SITE} NGINX_SERVERNAME=${NEW_SITE} fi if [ "${SITE_TYPE}" = "proxy" ]; then if [ ! -z ${PROXYSCHEME+x} ] && [ ! -z ${PROXYHOST+x} ] && [ ! -z ${PROXYPORT+x} ]; then GO=true; fi else GO=true; fi if [ "${GO}" = "true" ]; then echo -e "${idsCL[LightGreen]}Setting up new site for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}" echo "" if [ "${SITE_TYPE}" = "local" ]; then echo -e "server { listen 8080;" > /etc/nginx/sites-available/${MAIN_SITE}.conf if [ "${CREATE_SSL}" = "true" ]; then echo -e " listen 8443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf fi echo -e " server_name ${NGINX_SERVERNAME}; set \$base /var/www/${MAIN_SITE}; root \$base/public_html; access_log /var/log/nginx/${MAIN_SITE}-access.log; error_log /var/log/nginx/${MAIN_SITE}-error.log warn;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf if [ "${CREATE_SSL}" = "true" ]; then echo -e " ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem; include conf.d/include/ssl-ciphers.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf fi echo -e " index index.php; location / { try_files \$uri \$uri/ /index.php?\$query_string;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf if [ "${CREATE_SSL}" = "true" ]; then echo -e " include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf fi echo -e " } location ~ \.php\$ { fastcgi_pass unix:/var/run/php/php7.2-fpm.sock; include conf.d/include/php_fastcgi.conf; } include conf.d/include/general.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf if [ "${CREATE_SSL}" = "true" ]; then echo -e " include conf.d/include/letsencrypt-acme-challenge.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf fi echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE}.conf for nip in "${NODE_HOSTS[@]}" do if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' else NCMD="ssh root@${nip}" fi ${NCMD} mkdir -p /var/www/${MAIN_SITE}/{public_html,nginx_logs} done SET-PERMISSIONS ${MAIN_SITE} else echo -e "server { set \$forward_scheme ${PROXYSCHEME}; set \$server \"${PROXYHOST}\"; set \$port ${PROXYPORT}; listen 8080;" > /etc/nginx/sites-available/${MAIN_SITE}.conf if [ "${CREATE_SSL}" = "true" ]; then echo -e " listen 8443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf fi echo -e " server_name ${NGINX_SERVERNAME};" >> /etc/nginx/sites-available/${MAIN_SITE}.conf if [ "${CREATE_SSL}" = "true" ]; then echo -e " include conf.d/include/letsencrypt-acme-challenge.conf; include conf.d/include/ssl-ciphers.conf; ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf fi echo -e " access_log /var/log/nginx/proxy-${MAIN_SITE}.log proxy; location / {" >> /etc/nginx/sites-available/${MAIN_SITE}.conf if [ "${CREATE_SSL}" = "true" ]; then echo -e " include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}.conf fi echo -e " include conf.d/include/proxy.conf; } } " >> /etc/nginx/sites-available/${MAIN_SITE}.conf fi ln -s /etc/nginx/sites-available/${MAIN_SITE}.conf /etc/nginx/sites-enabled/${MAIN_SITE}.conf if [ "${CREATE_SSL}" = "true" ]; then NEWCERT ${NEW_SITE} else SERVICE nginx reload fi echo "" echo -e "${idsCL[LightGreen]}The new site for '${idsCL[LightGreen]}${NEW_SITE}${idsCL[Default]}' has been created.${idsCL[Default]}" echo "" else echo "Missing proxy arguments" exit 1 fi else echo "Missing arguments" echo "" echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {" width=33 printf "%-${width}s- %s\n" " -site {FQDN address(,es)}" "(*required)" printf "%-${width}s- %s\n" " -ssl {true or false}" "(defaults to true)" printf "%-${width}s- %s\n" " -type {'local' or 'proxy'}" "(defaults to local)" printf "%-${width}s- %s\n" " -scheme {http or https}" "(required if type set to proxy)" printf "%-${width}s- %s\n" " -host {IP or FQDN}" "(required if type set to proxy)" printf "%-${width}s- %s\n" " -port {host port}" "(required if type set to proxy)" echo "}" exit 1 fi } STATUS-CHECK(){ if [ "${STATUS_CHECK_EMAIL}" != "" ]; then if [ ! -z ${LOCAL_SERVICES+x} ]; then lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) for srvc in "${LOCAL_SERVICES[@]}" do if [ $(pgrep ${srvc} | wc -l) -lt "1" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} touch ${FOLDER}/localhost-${srvc}.down elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} rm -f ${FOLDER}/localhost-${srvc}.down fi done fi if [ -z ${LOCAL_SERVICES+x} ] || [ "${2}" = "all" ]; then nid=1 for nip in "${NODE_HOSTS[@]}" do if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' else NCMD="ssh root@${nip}" fi for srvc in "${NODE_SERVICES[@]}" do if [ $(${NCMD} pgrep ${srvc} | wc -l) -lt "1" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} touch ${FOLDER}/${nip}-${srvc}.down elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} rm -f ${FOLDER}/${nip}-${srvc}.down fi done nid=`expr $nid + 1` done fi fi } SET-PERMISSIONS(){ if [ "${1}" != "" ]; then NEW_SITE=${1} echo -e "${idsCL[Yellow]}Setting new site folder permissions for (/var/www/${NEW_SITE})${idsCL[Default]}" fi nid=1 for nip in "${NODE_HOSTS[@]}" do if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then nip='localhost ' NCMD='' else NCMD="ssh root@${nip}" fi if [ "${NEW_SITE}" != "" ]; then echo -en "${idsCL[Cyan]}iDS-Node${nid} (${nip}) ${idsCL[Default]}" ${NCMD} chown -R www-data:www-data /var/www/$NEW_SITE ${NCMD} chmod -R 7775 /var/www/$NEW_SITE echo -e "${idsCL[Green]}Complete${idsCL[Default]}" else echo -e "${idsCL[Yellow]}Setting folder permissions for iDS-Node${nid} (${nip})${idsCL[Default]}" echo -en "${idsCL[Cyan]}LetsEncrypt Certs ${idsCL[Default]}" ${NCMD} chown -R root:letsencrypt /etc/letsencrypt ${NCMD} chmod -R 6775 /etc/letsencrypt echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo -en "${idsCL[Cyan]}Webserver folders ${idsCL[Default]}" ${NCMD} chown -R www-data:www-data /var/www ${NCMD} chmod -R 7775 /var/www echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo -en "${idsCL[Cyan]}Gitea Folder ${idsCL[Default]}" ${NCMD} chown -R git:git /var/lib/gitea ${NCMD} chmod -R 750 /var/lib/gitea echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo "" fi nid=`expr $nid + 1` done if [ "${NEW_SITE}" != "" ]; then echo ""; fi if [ -z $action ] || [ "${action}" = "gui" ]; then ENTER2CONTINUE fi } SERVICE(){ if [ "${NM_SERVICES[${1}]}" = "" ]; then echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}" exit 1 fi if [[ "start,stop,restart,reload,enable,disable" != *"${2}"* ]] && [ "${1}" != "daemon-reload" ]; then echo -e "${idsCL[Red]}(${3}) is not an allowed service action.${idsCL[Default]}" exit 1 fi if [ "$2" = "stop" ]; then ADISP='Stopp' elif [ "$2" = "enable" ]; then ADISP='Enabl' elif [ "$2" = "disable" ]; then ADISP='Disabl' else ADISP=${2~} fi if [ "$2" = "start" ] && [ "$1" = "haproxy" ]; then TACT='restart'; else TACT="${2}"; fi echo -e "${idsCL[LightGreen]}[[${NM_SERVICES[${1}]} ${ADISP}ing]]${idsCL[Default]}" echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}" nid=1 for nip in "${NODE_HOSTS[@]}" do if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then nip='localhost ' NCMD='' else NCMD="ssh root@${nip}" fi echo -en "${ADISP}ing on iDS-Node${nid} ($nip)... ${idsCL[Default]}" $NCMD systemctl $2 $1 if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then echo -e "${idsCL[Green]}OK${idsCL[Default]}" elif [[ $(${NCMD} pgrep ${1} | wc -l) -gt "0" ]]; then echo -e "${idsCL[Green]}OK${idsCL[Default]}" elif [ "$2" = "stop" ]; then echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}" else echo -e "${idsCL[Red]}ERROR${idsCL[Default]}" fi nid=`expr $nid + 1` done echo "" echo -e "${idsCL[Green]}${NM_SERVICES[${1}]} has been ${ADISP}ed${idsCL[Default]}" echo "" } GUI(){ DISP_HEADER true true while : do echo " [1] Node Status" echo " [2] Node Services" echo "" echo " [3] New Site" echo " [4] New Certificate" echo " [5] Renew Certificates" echo " [6] List Certificates" echo "" echo " [7] Set folder permissions on nodes" echo "" echo " [Q] Quit" echo "" echo "" echo -e -n "${idsCL[LightYellow]}Please select an [ActionItem] from above:${idsCL[Default]} " read -n 1 opt echo "" case $opt in 1) STATUS; GUI;; 2) GUI;; 3) GUI;; 4) GUI;; 5) CERTRENEW; GUI;; 6) LISTCERTS; GUI;; 7) SET-PERMISSIONS; GUI;; [Qq]) EXIT1 exit 0;; *) echo "Thats an invaild option,"; echo "please select a valid option only."; sleep 1;; esac done } if [ ${action-x} ]; then DISP_HEADER case $action in newcert) NEWCERT ${2};; certrenew) CERTRENEW;; listcerts) LISTCERTS;; nightlyrenew) NIGHTLYRENEW;; delsite) DELSITE ${2};; newsite) NEWSITE ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14};; update) ;; status) STATUS ${2};; status-check) STATUS-CHECK ${2};; set-permissions) DIVIDER; SET-PERMISSIONS ${2};; service) SERVICE ${2} ${3};; gui) GUI;; *) echo -e "Usage: ${idsCL[Yellow]}nodemgmt${idsCL[Default]} { ${idsCL[Yellow]}service${idsCL[Default]} [service] [action] | ${idsCL[Yellow]}status${idsCL[Default]} | ${idsCL[Yellow]}newcert${idsCL[Default]} [domain(,s)] | ${idsCL[Yellow]}certrenew${idsCL[Default]} }" echo "" echo "" exit 0;; esac else GUI fi exit 0