#!/usr/bin/env bash VERS='3.03262019' NODETYPES=MYSQL,LB,WEB MYSQLHOSTS=10.5.10.51,10.5.10.52,10.5.10.53 WEBHOSTS=10.5.10.121,10.5.10.122,10.5.10.123 LBHOSTS=10.10.10.81,10.10.10.82,10.10.10.83 PRIMARYHOST=10.5.10.51 MYSQLMANAGER=10.5.10.50 NODESERVICES=mysql,nginx,gitea,powerdns-admin,haproxy,keepalived,maxscale,postgresql MYSQLSERVICES_CHECK=mysql WEBSERVICES_CHECK=nginx,gitea,keepalived LBSERVICES_CHECK=haproxy,keepalived noheader=' service status-check nightlyrenew backup report check checkcerts gitea ' CERT_DAEMON='/usr/bin/certbot' FOLDER=/opt/idssys/nodemgmt SCRIPT=${FOLDER}/nodemgmt-scripts.sh RENOTIFY=900 declare -i errtime IFS=, NODE_TYPES=(${NODETYPES}) MYSQL_HOSTS=(${MYSQLHOSTS}) WEB_HOSTS=(${WEBHOSTS}) LB_HOSTS=(${LBHOSTS}) NODE_SERVICES=(${NODESERVICES}) MYSQL_SERVICES_CHECK=(${MYSQLSERVICES_CHECK}) WEB_SERVICES_CHECK=(${WEBSERVICES_CHECK}) LB_SERVICES_CHECK=(${LBSERVICES_CHECK}) LOCAL_SERVICES=(${LOCAL_SERVICES}) unset IFS declare -A NM_NODETYPES NM_NODETYPES['MYSQL']='MySQL' NM_NODETYPES['LB']='LoadBalance' NM_NODETYPES['WEB']='Webserver' declare -A NM_SERVICES NM_SERVICES['mysql']='MySQL' NM_SERVICES['postgresql']='PostgreSQL' NM_SERVICES['nginx']='NGINX' NM_SERVICES['gogs']='Gogs' NM_SERVICES['gitea']='Gitea' NM_SERVICES['powerdns-admin']='PowerDNS-Admin' NM_SERVICES['haproxy']='HAProxy' NM_SERVICES['keepalived']='Keepalived' NM_SERVICES['maxscale']='MaxScale' NM_SERVICES['cmon']='CC-Controller' NM_SERVICES['cmon-events']='CC-Events Manager' NM_SERVICES['cmon-cloud']='CC-Cloud Daemon' # NM_SERVICES['powerdns-admin']='PowerDNS-Admin' declare -A NM_SRVCOPTS NM_SRVCOPTS['start']='Start' NM_SRVCOPTS['stop']='Stopp' NM_SRVCOPTS['restart']='Restart' NM_SRVCOPTS['reload']='Reload' NM_SRVCOPTS['enable']='Enabl' NM_SRVCOPTS['disable']='Disabl' NM_SRVCOPTS['daemon-reload']='Daemon-Reload' NM_SRVCOPT=(start stop restart reload enable disable) declare -A REPL_CHECKS REPL_CHECKS['nginx']='/etc/nginx' REPL_CHECKS['ssl']='/etc/letsencrypt' REPL_CHECKS['lessh']='/home/le/.ssh' REPL_CHECKS['php']='/etc/php' REPL_CHECKS['www']='/var/www' REPL_CHECKS['git']='/var/lib/gitea' REPL_CHECKS['gitssh']='/home/git' REPL_CHECKS['pma']='/etc/phpmyadmin' MYSQL_REPL_CHECK=( ) WEB_REPL_CHECK=(nginx www ssl lessh git gitssh php pma) LB_REPL_CHECK=( ) declare -A REPL_DESC REPL_DESC['nginx']='NGINX Settings' REPL_DESC['ssl']='SSL Certificates' REPL_DESC['lessh']='LetsEncrypt SSH KEys' REPL_DESC['php']='PHP Settings' REPL_DESC['www']='Webserver Files' REPL_DESC['git']='Gitea System' REPL_DESC['gitssh']='Gitea SSH Keys' REPL_DESC['pma']='phpMyAdmin Settings' declare -A BACKUP_ITEMS BACKUP_ITEMS[nginx-settings]=/etc/nginx BACKUP_ITEMS[letsencrypt-certs]=/etc/letsencrypt BACKUP_ITEMS[gitea]=/var/lib/gitea BACKUP_ITEMS[nginx-logs]='/var/www/!NGINX-Logs' #BACKUP_ITEMS[webserver-files]=/var/www GET-CHECKCERT-DOMAINS(){ declare -A CHECKCERT_DOMAINS IFS=$'\n' for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do HOST=${LINE%% *} PORT=${LINE#* } IFS=" " CHECKCERT_DOMAINS[${HOST}]=${PORT} done unset IFS } DISP_HEADER(){ declare -i cw; declare -i spc1; declare -i c if [ "$1" = true ]; then clear fi echo "" echo -e "${idsCL[LightGreen]} NodeMgmt - Galera/NGINX Node Management${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${idsCL[Default]}" DIVIDER . lightGreen if [ "$2" = true ]; then echo -e " Node hostname : ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME}${idsCL[Default]}${idsST[Reset]}" if [ -z ${LOCAL_SERVICES+x} ]; then SERVICES=$( IFS=$','; echo "${NODE_SERVICES[*]}" ) else SERVICES=$( IFS=$','; echo "${LOCAL_SERVICES[*]}" ) fi IFS=,; SERVICES2=(${SERVICES}); unset IFS for srvc in "${SERVICES2[@]}"; do c=0; cw=18; spc='' spc1=${cw}-${#NM_SERVICES[${srvc}]} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en " ${NM_SERVICES[${srvc}]}$spc: " if [ $(pgrep ${srvc} | wc -l) -gt "0" ]; then echo -e "${idsCL[Green]}Running${idsCL[Default]}" else echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi done DIVIDER if [ $(ls -1 ${FOLDER}/*.lastrun 2>/dev/null | wc -l) != 0 ];then echo -e "${idsST[Bold]}Lastrun Items:${idsST[Reset]}" for lastrun in ${FOLDER}/*.lastrun ; do IFS='/'; lastrun_item=(${lastrun}); unset IFS lastrun_item=$(echo ${lastrun_item[4]} | sed "s/.lastrun//g") lastrun_item=$(echo ${lastrun_item} | sed "s/-/ /g") lastrun_date=$(stat -c %y ${lastrun}) IFS=' '; lastrun_date=(${lastrun_date}); unset IFS IFS='.'; lastrun_time=(${lastrun_date[1]}); unset IFS c=0; cw=18; spc='' spc1=${cw}-${#lastrun_item} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -e " ${lastrun_item~}${spc}: ${lastrun_date[0]} ${lastrun_time}" done DIVIDER fi fi echo "" } DIVIDER(){ if [ -z ${2+x} ]; then local clr='yellow' else local clr=$2 fi if [ -z ${3+x} ]; then local length=70 else local length=$3 fi local c=0; local dashes=''; until [ $c = ${length} ]; do local dashes="${dashes}-"; local c=`expr $c + 1`; done echo -e "${idsCL[${clr~}]}${dashes}${idsCL[Default]}" if [ "$1" = true ]; then echo "" fi } CERT-CHECK(){ TARGET="mysite.example.net"; RECIPIENT="hostmaster@mysite.example.net"; DAYS=7; echo "checking if $TARGET expires in less than $DAYS days"; expirationdate=$(date -d "$(: | openssl s_client -connect $TARGET:443 -servername $TARGET 2>/dev/null \ | openssl x509 -text \ | grep 'Not After' \ |awk '{print $4,$5,$7}')" '+%s'); in7days=$(($(date +%s) + (86400*$DAYS))); if [ $in7days -gt $expirationdate ]; then echo "KO - Certificate for $TARGET expires in less than $DAYS days, on $(date -d @$expirationdate '+%Y-%m-%d')" \ | mail -s "Certificate expiration warning for $TARGET" $RECIPIENT ; else echo "OK - Certificate expires on $expirationdate"; fi } CHECK_HOST(){ if [ ! -z ${1+x} ]; then ping -c 3 ${1} > /dev/null 2>&1 if [ $? -ne 0 ]; then echo false else echo true fi else echo false fi }