From 1ab1de4132c82547941f998fd83cf22c3cbfef7c Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Mon, 3 Jul 2023 23:18:52 -0500 Subject: [PATCH] update --- defaults.inc | 16 ++++++++++- defaults.local.inc | 42 ++++++++------------------- inc/status.inc | 70 ++++++++++++++++++++++----------------------- nodemgmt-scripts.sh | 14 ++++----- 4 files changed, 69 insertions(+), 73 deletions(-) diff --git a/defaults.inc b/defaults.inc index 13140e45..879a582b 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ -VERS='4.7.160-07032023' +VERS='4.7.170-07032023' noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns ' CERT_DAEMON='/snap/bin/certbot' @@ -19,6 +19,20 @@ declare -i errtime NM_SRVCOPT=(start stop restart reload enable disable) if [ -f ${FOLDER}/defaults.local.inc ]; then + declare -A NM_HOSTS + declare -A NM_SERVICES_CHECK + declare -A NM_DOCKERS_CHECK + declare -A NM_REPL_CHECK + declare -A NGINX_PATHS + declare -A NM_HOSTNAMES + declare -A NM_NODETYPES + declare -A NM_SERVICE_DESC + declare -A NM_DOCKER_DESC + declare -A NM_DOCKER_COMPOSE_LOC + declare -A NM_REPL_CHECK_LOC + declare -A NM_REPL_DESC + + source ${FOLDER}/defaults.local.inc IFS=, diff --git a/defaults.local.inc b/defaults.local.inc index 8c32162c..4a850eff 100755 --- a/defaults.local.inc +++ b/defaults.local.inc @@ -1,20 +1,16 @@ +NM_NODEMANAGER=10.10.10.50 -# NODETYPES=WEB +# declare -A NM_NEW_ARRAY +# NM_NEW_ARRAY['LB']= +# NM_NEW_ARRAY['LPD']= +# NM_NEW_ARRAY['MM']= +# NM_NEW_ARRAY['MYSQL']= +# NM_NEW_ARRAY['NC']= +# NM_NEW_ARRAY['PW']= +# NM_NEW_ARRAY['WPD']= +# NM_NEW_ARRAY['WM']= +# NM_NEW_ARRAY['WEB']= -NODEMANAGER=10.10.10.50 - -# declare -A NM_HOST_SERVICES -# NM_HOST_SERVICES['LB']= -# NM_HOST_SERVICES['LPD']= -# NM_HOST_SERVICES['MM']= -# NM_HOST_SERVICES['MYSQL']= -# NM_HOST_SERVICES['NC']= -# NM_HOST_SERVICES['PW']= -# NM_HOST_SERVICES['WPD']= -# NM_HOST_SERVICES['WM']= -# NM_HOST_SERVICES['WEB']= - -declare -A NM_HOSTS # NM_HOSTS['GL']=10.10.1.41,10.10.1.42,10.10.1.121,10.10.1.122 NM_HOSTS['LB']=10.10.10.81,10.10.10.82 NM_HOSTS['LPD']=10.10.10.10,10.10.10.11,10.2.1.2 @@ -26,7 +22,6 @@ NM_HOSTS['WPD']=100.100.55.2,100.100.55.3,10.10.1.28 NM_HOSTS['WM']=100.100.55.5 NM_HOSTS['WEB']=10.10.1.121,10.10.1.122 -declare -A NM_SERVICES_CHECK NM_SERVICES_CHECK['LB']=nginx,haproxy,keepalived NM_SERVICES_CHECK['LPD']=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,keepalived NM_SERVICES_CHECK['MM']=bazarr,radarr,sonarr,prowlarr,qbittorrent,snap.tautulli.tautulli,ombi,nginx,php8.2-fpm @@ -37,7 +32,6 @@ NM_SERVICES_CHECK['WPD']=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,proxysql, NM_SERVICES_CHECK['WM']=nginx,tailscaled NM_SERVICES_CHECK['WEB']=nginx,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,php8.2-fpm,gitea,keepalived -declare -A NM_DOCKERS_CHECK NM_DOCKERS_CHECK['LB']= NM_DOCKERS_CHECK['LPD']= NM_DOCKERS_CHECK['MM']= @@ -48,7 +42,6 @@ NM_DOCKERS_CHECK['WPD']= NM_DOCKERS_CHECK['WM']=uptime-kuma NM_DOCKERS_CHECK['WEB']=phpmyadmin,phppgadmin,vaultwarden,heimdall,authelia -declare -A NM_REPL_CHECK NM_REPL_CHECK['LB']=lb-data NM_REPL_CHECK['LPD']= NM_REPL_CHECK['MM']= @@ -70,11 +63,9 @@ LB_NGINX_SITES='/opt/lb-data/nginx/sites-enabled' WEB_NGINX_SITES='/mnt/web-data/nginx/sites-enabled' -declare -A NGINX_PATHS NGINX_PATHS['LB']='/opt/lb-data/nginx' NGINX_PATHS['WEB']='/mnt/web-data/nginx' -declare -A NM_HOSTNAMES NM_HOSTNAMES['10.10.1.121']='Webserver-Node1' NM_HOSTNAMES['10.10.1.122']='Webserver-Node2' NM_HOSTNAMES['10.10.1.24']='Media-Manager' @@ -90,13 +81,9 @@ NM_HOSTNAMES['10.10.10.50']='MySQL-Manager' NM_HOSTNAMES['10.10.0.240']='PowerWall-Server' NM_HOSTNAMES['100.100.55.2']='WAN-DNS-Node1 (East)' NM_HOSTNAMES['100.100.55.3']='WAN-DNS-Node2 (West)' -NM_HOSTNAMES['100.100.55.5']='WAN-Monitor-Node1' -NM_HOSTNAMES['3.214.174.170']='WAN-DNS-Node1 (East)' -NM_HOSTNAMES['54.213.11.227']='WAN-DNS-Node2 (West)' NM_HOSTNAMES['10.10.1.28']='WAN-DNS-Node3 (Home)' -NM_HOSTNAMES['44.214.59.192']='WAN-Monitor-Node1' +NM_HOSTNAMES['100.100.55.5']='WAN-Monitor-Node1' -declare -A NM_NODETYPES NM_NODETYPES['MYSQL']='MySQL' NM_NODETYPES['LB']='LoadBalance' NM_NODETYPES['WEB']='Webserver' @@ -108,7 +95,6 @@ NM_NODETYPES['PW']='PowerWall' NM_NODETYPES['WM']='WAN-Monitor' NM_NODETYPES['GL']='Gluster' -declare -A NM_SERVICE_DESC NM_SERVICE_DESC['mysql']='MySQL' NM_SERVICE_DESC['mariadb']='MariaDB (MySQL)' NM_SERVICE_DESC['postgresql']='PostgreSQL' @@ -149,7 +135,6 @@ NM_SERVICE_DESC['monitor-power']='Server Power Monitor' NM_SERVICE_DESC['monitor-temp']='Server Temp Monitor' -declare -A NM_DOCKER_DESC NM_DOCKER_DESC['authelia']='Authelia' NM_DOCKER_DESC['phpmyadmin']='phpMyAdmin' NM_DOCKER_DESC['phppgadmin']='phpPgAdmin' @@ -158,7 +143,6 @@ NM_DOCKER_DESC['heimdall']='Heimdall' NM_DOCKER_DESC['uptime-kuma']='Uptime-Kuma' NM_DOCKER_DESC['nginx-proxy_manager']='Nginx Proxy Manager' -declare -A NM_DOCKER_COMPOSE_LOC NM_DOCKER_COMPOSE_LOC['authelia']='/mnt/web-data/authelia' NM_DOCKER_COMPOSE_LOC['phpmyadmin']='/mnt/web-data/phpmyadmin' NM_DOCKER_COMPOSE_LOC['phppgadmin']='/mnt/web-data/phppgadmin' @@ -167,7 +151,6 @@ NM_DOCKER_COMPOSE_LOC['heimdall']='/mnt/web-data/heimdall' NM_DOCKER_COMPOSE_LOC['uptime-kuma']='/opt/uptime-kuma' NM_DOCKER_COMPOSE_LOC['nginx-proxy_manager']='/opt/nginx-proxy' -declare -A NM_REPL_CHECK_LOC NM_REPL_CHECK_LOC['apache']='/etc/apache2' NM_REPL_CHECK_LOC['nginx']='/mnt/web-data/nginx' NM_REPL_CHECK_LOC['nginxproxy']='/opt/nginx-proxy' @@ -184,7 +167,6 @@ NM_REPL_CHECK_LOC['pdnsa']='/mnt/web-data/powerdns-admin' NM_REPL_CHECK_LOC['nextcloud-data']='/mnt/Nextcloud-Data' NM_REPL_CHECK_LOC['nextcloud-php']='/etc/php' -declare -A NM_REPL_DESC NM_REPL_DESC['apache']='Apache Settings' NM_REPL_DESC['nginx']='NGINX Settings' NM_REPL_DESC['nginxproxy']='NGINX-Proxy Settings' diff --git a/inc/status.inc b/inc/status.inc index 1b34e46a..50bdef18 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -142,7 +142,7 @@ STATUS(){ DIVIDER; echo -e "${idsST[Reset]}" fi for nip in "${!var}"; do - [ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}" + [ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}" if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' else NCMD="ssh root@${nip}"; LH='' @@ -152,7 +152,7 @@ STATUS(){ fi if [ "${checkhost}" != "false" ]; then if [ "${ST_ACTION}" != "check" ]; then - #echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}" + #echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}" if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then uptime=`${NCMD} uptime -p` echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" @@ -169,7 +169,7 @@ STATUS(){ toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) fi rm -f ${TMPFOLDER}/${nip}.* - SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}-UP" "${NM_HOSTNAMES[${nip}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" fi if [ "${ST_ACTION}" != "dockers" ]; then @@ -201,19 +201,19 @@ STATUS(){ if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" touch ${TMPFOLDER}/${nip}~${srvc}.down - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE} + # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE} else errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down) if [ $errtime -gt ${RENOTIFY} ]; then [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) touch ${TMPFOLDER}/${nip}~${srvc}.down - SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is down" 1 + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is down" 1 + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 fi [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" @@ -222,8 +222,8 @@ STATUS(){ [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then [ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down) - SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} rm -f ${TMPFOLDER}/${nip}~${srvc}.* @@ -272,9 +272,9 @@ STATUS(){ # if [ "${ST_ACTION}" != "check" ]; then # echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" # fi - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is down" 1 + # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 # touch ${TMPFOLDER}/${nip}~${srvc}.down - # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE} + # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE} # # else # errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down) @@ -302,8 +302,8 @@ STATUS(){ # else # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down) # fi - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} + # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} # # rm -f ${TMPFOLDER}/${nip}~${srvc}.down # rm -f ${TMPFOLDER}/${nip}~${srvc}.errtime @@ -347,28 +347,28 @@ STATUS(){ if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then ${NCMD} docker start ${docker} >/dev/null 2>&1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE} else ${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE} fi sleep 10s if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" fi - SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is fixed" + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed" rm -f ${TMPFOLDER}/${nip}~${docker}.* - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE} else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1 + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1 touch ${TMPFOLDER}/${nip}~${docker}.down - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE} else errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down) if [ $errtime -gt ${RENOTIFY} ]; then @@ -405,8 +405,8 @@ STATUS(){ toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" fi - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_HOSTNAMES[${nip}]} is down" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - Node is down" >> ${LOGFILE} + # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${LOGFILE} else toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) @@ -415,12 +415,12 @@ STATUS(){ fi if [ $errtime -gt ${RENOTIFY} ]; then touch ${TMPFOLDER}/${nip}.down - SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_HOSTNAMES[${nip}]} has been down for $(SHOW_TIME ${toterrtime})" 1 + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1 elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_HOSTNAMES[${nip}]} is down" 1 + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 fi # if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then - # echo "${NM_HOSTNAMES[${nip}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_HOSTNAMES[${nip}]}-${nip}" ${EMAIL_NOTICE} + # echo "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_HOSTNAMES[${nip}]}[${nip}]" ${EMAIL_NOTICE} # fi fi @@ -474,7 +474,7 @@ STATUS(){ for nip in "${!var}"; do if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[Default]}" + echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}" DIVIDER false green fi if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then @@ -504,22 +504,22 @@ STATUS(){ [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then touch ${TMPFOLDER}/${nip}~${rcheck}.down - SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} + SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1 + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} fi else [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}" if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then rm -f ${TMPFOLDER}/${nip}~${rcheck}.down - SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} + SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} fi fi done else if [ "${ST_ACTION}" != "check" ]; then [ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}" - [ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]} (${nip}) is offline${idsCL[Default]}" + [ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}" fi fi if [ "${ST_ACTION}" != "check" ]; then echo; fi @@ -584,7 +584,7 @@ STATUS(){ else NCMD="ssh root@${nip}"; LH='' fi if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}" + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}" DIVIDER false green fi @@ -617,11 +617,11 @@ STATUS(){ if [ "${freespace}" -le "1024" ]; then fs_status='error' fs_status_color='Red' - SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}'" "${partition} : ${fsdsp} free" 1 + SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free" 1 elif [ "${freespace}" -le "5120" ]; then fs_status='warn' fs_status_color='Yellow' - SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}'" "${partition} : ${fsdsp} free" + SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free" else fs_status='' fs_status_color='Green' @@ -647,12 +647,12 @@ STATUS(){ # if [ "${freespace}" -le "1000" ]; then # fs_status='error' # fs_status_color='Red' - # SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}'" "${partition} : ${freespace} MB free" 1 + # SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free" 1 # # elif [ "${freespace}" -le "5000" ]; then # fs_status='warn' # fs_status_color='Yellow' - # SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}'" "${partition} : ${freespace} MB free" + # SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free" # # else # fs_status='' diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 20d49172..87dd5fbd 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -35,7 +35,7 @@ CHECK_DOCKER_SERVICES(){ fi if [ "${checkhost}" != "false" ]; then if [ "${ST_ACTION}" != "check" ]; then - echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}" + echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}" uptime=`${NCMD} uptime -p` echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" DIVIDER false green @@ -48,7 +48,7 @@ CHECK_DOCKER_SERVICES(){ fi rm -f ${TMPFOLDER}/${nip}.down rm -f ${TMPFOLDER}/${nip}.errtime - SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}-UP" "${NM_HOSTNAMES[${nip}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" fi for docker in "${!dockers}"; do @@ -90,7 +90,7 @@ CHECK_DOCKER_SERVICES(){ fi if [ "${ST_ACTION}" != "check" ]; then toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" fi fi if [ "${ST_ACTION}" != "check" ]; then echo; fi @@ -528,7 +528,7 @@ SERVICE(){ if [ $sethost == true ]; then nodename=${NM_HOSTNAMES[${3}]} else - nodename="${NM_HOSTNAMES[${nip}]}" + nodename="${NM_HOSTNAMES[${nip}]}[${nip}]" fi if [[ "${MIP}" == *"${nip}"* ]]; then nip='localhost ' @@ -778,7 +778,7 @@ NODEUPDATE() { declare -i cw; declare -i spc1; declare -i c echo -e "${idsCL[LightGreen]}Updating All Nodes${idsCL[Default]}" echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}" - nip=${NODEMANAGER} + nip=${NM_NODEMANAGER} if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then nip='localhost' fi @@ -808,7 +808,7 @@ NODEUPDATE() { c=0; cw=15; spcb='' spc1=${cw}-${#nip} until [ $c = ${spc1} ]; do spcb="${spcb} "; c=`expr $c + 1`; done - echo -en "Updating ${NM_HOSTNAMES[${nip}]}${spca}($nip)${spcb}... ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : " + echo -en "Updating ${NM_HOSTNAMES[${nip}]}[${nip}]${spca}($nip)${spcb}... ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : " ${NCMD} "nodemgmt update q" echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ... ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : " ${NCMD} "runup runup" @@ -994,7 +994,7 @@ RUN_COMMAND(){ if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' else NCMD="ssh root@${nip}"; LH='' fi - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}" + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}" DIVIDER false green #echo "Running command: ${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}" ${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}