voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-07-03 23:18:52 -05:00
parent c1fdda6e7d
commit 1ab1de4132
4 changed files with 69 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
defaults.inc
View File

@@ -1,5 +1,5 @@
VERS='4.7.160-07032023'
VERS='4.7.170-07032023'
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns '
CERT_DAEMON='/snap/bin/certbot'
@@ -19,6 +19,20 @@ declare -i errtime
NM_SRVCOPT=(start stop restart reload enable disable)
if [ -f ${FOLDER}/defaults.local.inc ]; then
declare -A NM_HOSTS
declare -A NM_SERVICES_CHECK
declare -A NM_DOCKERS_CHECK
declare -A NM_REPL_CHECK
declare -A NGINX_PATHS
declare -A NM_HOSTNAMES
declare -A NM_NODETYPES
declare -A NM_SERVICE_DESC
declare -A NM_DOCKER_DESC
declare -A NM_DOCKER_COMPOSE_LOC
declare -A NM_REPL_CHECK_LOC
declare -A NM_REPL_DESC
source ${FOLDER}/defaults.local.inc
IFS=,

42
defaults.local.inc
View File

@@ -1,20 +1,16 @@
NM_NODEMANAGER=10.10.10.50
# NODETYPES=WEB
# declare -A NM_NEW_ARRAY
# NM_NEW_ARRAY['LB']=
# NM_NEW_ARRAY['LPD']=
# NM_NEW_ARRAY['MM']=
# NM_NEW_ARRAY['MYSQL']=
# NM_NEW_ARRAY['NC']=
# NM_NEW_ARRAY['PW']=
# NM_NEW_ARRAY['WPD']=
# NM_NEW_ARRAY['WM']=
# NM_NEW_ARRAY['WEB']=
NODEMANAGER=10.10.10.50
# declare -A NM_HOST_SERVICES
# NM_HOST_SERVICES['LB']=
# NM_HOST_SERVICES['LPD']=
# NM_HOST_SERVICES['MM']=
# NM_HOST_SERVICES['MYSQL']=
# NM_HOST_SERVICES['NC']=
# NM_HOST_SERVICES['PW']=
# NM_HOST_SERVICES['WPD']=
# NM_HOST_SERVICES['WM']=
# NM_HOST_SERVICES['WEB']=
declare -A NM_HOSTS
# NM_HOSTS['GL']=10.10.1.41,10.10.1.42,10.10.1.121,10.10.1.122
NM_HOSTS['LB']=10.10.10.81,10.10.10.82
NM_HOSTS['LPD']=10.10.10.10,10.10.10.11,10.2.1.2
@@ -26,7 +22,6 @@ NM_HOSTS['WPD']=100.100.55.2,100.100.55.3,10.10.1.28
NM_HOSTS['WM']=100.100.55.5
NM_HOSTS['WEB']=10.10.1.121,10.10.1.122
declare -A NM_SERVICES_CHECK
NM_SERVICES_CHECK['LB']=nginx,haproxy,keepalived
NM_SERVICES_CHECK['LPD']=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,keepalived
NM_SERVICES_CHECK['MM']=bazarr,radarr,sonarr,prowlarr,qbittorrent,snap.tautulli.tautulli,ombi,nginx,php8.2-fpm
@@ -37,7 +32,6 @@ NM_SERVICES_CHECK['WPD']=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,proxysql,
NM_SERVICES_CHECK['WM']=nginx,tailscaled
NM_SERVICES_CHECK['WEB']=nginx,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,php8.2-fpm,gitea,keepalived
declare -A NM_DOCKERS_CHECK
NM_DOCKERS_CHECK['LB']=
NM_DOCKERS_CHECK['LPD']=
NM_DOCKERS_CHECK['MM']=
@@ -48,7 +42,6 @@ NM_DOCKERS_CHECK['WPD']=
NM_DOCKERS_CHECK['WM']=uptime-kuma
NM_DOCKERS_CHECK['WEB']=phpmyadmin,phppgadmin,vaultwarden,heimdall,authelia
declare -A NM_REPL_CHECK
NM_REPL_CHECK['LB']=lb-data
NM_REPL_CHECK['LPD']=
NM_REPL_CHECK['MM']=
@@ -70,11 +63,9 @@ LB_NGINX_SITES='/opt/lb-data/nginx/sites-enabled'
WEB_NGINX_SITES='/mnt/web-data/nginx/sites-enabled'
declare -A NGINX_PATHS
NGINX_PATHS['LB']='/opt/lb-data/nginx'
NGINX_PATHS['WEB']='/mnt/web-data/nginx'
declare -A NM_HOSTNAMES
NM_HOSTNAMES['10.10.1.121']='Webserver-Node1'
NM_HOSTNAMES['10.10.1.122']='Webserver-Node2'
NM_HOSTNAMES['10.10.1.24']='Media-Manager'
@@ -90,13 +81,9 @@ NM_HOSTNAMES['10.10.10.50']='MySQL-Manager'
NM_HOSTNAMES['10.10.0.240']='PowerWall-Server'
NM_HOSTNAMES['100.100.55.2']='WAN-DNS-Node1 (East)'
NM_HOSTNAMES['100.100.55.3']='WAN-DNS-Node2 (West)'
NM_HOSTNAMES['100.100.55.5']='WAN-Monitor-Node1'
NM_HOSTNAMES['3.214.174.170']='WAN-DNS-Node1 (East)'
NM_HOSTNAMES['54.213.11.227']='WAN-DNS-Node2 (West)'
NM_HOSTNAMES['10.10.1.28']='WAN-DNS-Node3 (Home)'
NM_HOSTNAMES['44.214.59.192']='WAN-Monitor-Node1'
NM_HOSTNAMES['100.100.55.5']='WAN-Monitor-Node1'
declare -A NM_NODETYPES
NM_NODETYPES['MYSQL']='MySQL'
NM_NODETYPES['LB']='LoadBalance'
NM_NODETYPES['WEB']='Webserver'
@@ -108,7 +95,6 @@ NM_NODETYPES['PW']='PowerWall'
NM_NODETYPES['WM']='WAN-Monitor'
NM_NODETYPES['GL']='Gluster'
declare -A NM_SERVICE_DESC
NM_SERVICE_DESC['mysql']='MySQL'
NM_SERVICE_DESC['mariadb']='MariaDB (MySQL)'
NM_SERVICE_DESC['postgresql']='PostgreSQL'
@@ -149,7 +135,6 @@ NM_SERVICE_DESC['monitor-power']='Server Power Monitor'
NM_SERVICE_DESC['monitor-temp']='Server Temp Monitor'
declare -A NM_DOCKER_DESC
NM_DOCKER_DESC['authelia']='Authelia'
NM_DOCKER_DESC['phpmyadmin']='phpMyAdmin'
NM_DOCKER_DESC['phppgadmin']='phpPgAdmin'
@@ -158,7 +143,6 @@ NM_DOCKER_DESC['heimdall']='Heimdall'
NM_DOCKER_DESC['uptime-kuma']='Uptime-Kuma'
NM_DOCKER_DESC['nginx-proxy_manager']='Nginx Proxy Manager'
declare -A NM_DOCKER_COMPOSE_LOC
NM_DOCKER_COMPOSE_LOC['authelia']='/mnt/web-data/authelia'
NM_DOCKER_COMPOSE_LOC['phpmyadmin']='/mnt/web-data/phpmyadmin'
NM_DOCKER_COMPOSE_LOC['phppgadmin']='/mnt/web-data/phppgadmin'
@@ -167,7 +151,6 @@ NM_DOCKER_COMPOSE_LOC['heimdall']='/mnt/web-data/heimdall'
NM_DOCKER_COMPOSE_LOC['uptime-kuma']='/opt/uptime-kuma'
NM_DOCKER_COMPOSE_LOC['nginx-proxy_manager']='/opt/nginx-proxy'
declare -A NM_REPL_CHECK_LOC
NM_REPL_CHECK_LOC['apache']='/etc/apache2'
NM_REPL_CHECK_LOC['nginx']='/mnt/web-data/nginx'
NM_REPL_CHECK_LOC['nginxproxy']='/opt/nginx-proxy'
@@ -184,7 +167,6 @@ NM_REPL_CHECK_LOC['pdnsa']='/mnt/web-data/powerdns-admin'
NM_REPL_CHECK_LOC['nextcloud-data']='/mnt/Nextcloud-Data'
NM_REPL_CHECK_LOC['nextcloud-php']='/etc/php'
declare -A NM_REPL_DESC
NM_REPL_DESC['apache']='Apache Settings'
NM_REPL_DESC['nginx']='NGINX Settings'
NM_REPL_DESC['nginxproxy']='NGINX-Proxy Settings'

70
inc/status.inc
View File

@@ -142,7 +142,7 @@ STATUS(){
DIVIDER; echo -e "${idsST[Reset]}"
fi
for nip in "${!var}"; do
[ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}"
[ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}"
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
@@ -152,7 +152,7 @@ STATUS(){
fi
if [ "${checkhost}" != "false" ]; then
if [ "${ST_ACTION}" != "check" ]; then
#echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}"
#echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}"
if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then
uptime=`${NCMD} uptime -p`
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
@@ -169,7 +169,7 @@ STATUS(){
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
fi
rm -f ${TMPFOLDER}/${nip}.*
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}-UP" "${NM_HOSTNAMES[${nip}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
if [ "${ST_ACTION}" != "dockers" ]; then
@@ -201,19 +201,19 @@ STATUS(){
if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
touch ${TMPFOLDER}/${nip}~${srvc}.down
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
[ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
touch ${TMPFOLDER}/${nip}~${srvc}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
fi
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
@@ -222,8 +222,8 @@ STATUS(){
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
rm -f ${TMPFOLDER}/${nip}~${srvc}.*
@@ -272,9 +272,9 @@ STATUS(){
# if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# touch ${TMPFOLDER}/${nip}~${srvc}.down
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
#
# else
# errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
@@ -302,8 +302,8 @@ STATUS(){
# else
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
#
# rm -f ${TMPFOLDER}/${nip}~${srvc}.down
# rm -f ${TMPFOLDER}/${nip}~${srvc}.errtime
@@ -347,28 +347,28 @@ STATUS(){
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
else
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
fi
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is fixed"
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed"
rm -f ${TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
touch ${TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
@@ -405,8 +405,8 @@ STATUS(){
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_HOSTNAMES[${nip}]} is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - Node is down" >> ${LOGFILE}
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${LOGFILE}
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
@@ -415,12 +415,12 @@ STATUS(){
fi
if [ $errtime -gt ${RENOTIFY} ]; then
touch ${TMPFOLDER}/${nip}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_HOSTNAMES[${nip}]} has been down for $(SHOW_TIME ${toterrtime})" 1
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_HOSTNAMES[${nip}]} is down" 1
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
fi
# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then
# echo "${NM_HOSTNAMES[${nip}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_HOSTNAMES[${nip}]}-${nip}" ${EMAIL_NOTICE}
# echo "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_HOSTNAMES[${nip}]}[${nip}]" ${EMAIL_NOTICE}
# fi
fi
@@ -474,7 +474,7 @@ STATUS(){
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[Default]}"
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
@@ -504,22 +504,22 @@ STATUS(){
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi
else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi
fi
done
else
if [ "${ST_ACTION}" != "check" ]; then
[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
[ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]} (${nip}) is offline${idsCL[Default]}"
[ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
@@ -584,7 +584,7 @@ STATUS(){
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
@@ -617,11 +617,11 @@ STATUS(){
if [ "${freespace}" -le "1024" ]; then
fs_status='error'
fs_status_color='Red'
SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}'" "${partition} : ${fsdsp} free" 1
SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free" 1
elif [ "${freespace}" -le "5120" ]; then
fs_status='warn'
fs_status_color='Yellow'
SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}'" "${partition} : ${fsdsp} free"
SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free"
else
fs_status=''
fs_status_color='Green'
@@ -647,12 +647,12 @@ STATUS(){
# if [ "${freespace}" -le "1000" ]; then
# fs_status='error'
# fs_status_color='Red'
# SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}'" "${partition} : ${freespace} MB free" 1
# SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free" 1
#
# elif [ "${freespace}" -le "5000" ]; then
# fs_status='warn'
# fs_status_color='Yellow'
# SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}'" "${partition} : ${freespace} MB free"
# SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free"
#
# else
# fs_status=''

14
nodemgmt-scripts.sh
View File

@@ -35,7 +35,7 @@ CHECK_DOCKER_SERVICES(){
fi
if [ "${checkhost}" != "false" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}"
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}"
uptime=`${NCMD} uptime -p`
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
@@ -48,7 +48,7 @@ CHECK_DOCKER_SERVICES(){
fi
rm -f ${TMPFOLDER}/${nip}.down
rm -f ${TMPFOLDER}/${nip}.errtime
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}-UP" "${NM_HOSTNAMES[${nip}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
for docker in "${!dockers}"; do
@@ -90,7 +90,7 @@ CHECK_DOCKER_SERVICES(){
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
@@ -528,7 +528,7 @@ SERVICE(){
if [ $sethost == true ]; then
nodename=${NM_HOSTNAMES[${3}]}
else
nodename="${NM_HOSTNAMES[${nip}]}"
nodename="${NM_HOSTNAMES[${nip}]}[${nip}]"
fi
if [[ "${MIP}" == *"${nip}"* ]]; then
nip='localhost '
@@ -778,7 +778,7 @@ NODEUPDATE() {
declare -i cw; declare -i spc1; declare -i c
echo -e "${idsCL[LightGreen]}Updating All Nodes${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
nip=${NODEMANAGER}
nip=${NM_NODEMANAGER}
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
nip='localhost'
fi
@@ -808,7 +808,7 @@ NODEUPDATE() {
c=0; cw=15; spcb=''
spc1=${cw}-${#nip}
until [ $c = ${spc1} ]; do spcb="${spcb} "; c=`expr $c + 1`; done
echo -en "Updating ${NM_HOSTNAMES[${nip}]}${spca}($nip)${spcb}... ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
echo -en "Updating ${NM_HOSTNAMES[${nip}]}[${nip}]${spca}($nip)${spcb}... ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
${NCMD} "nodemgmt update q"
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ... ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
${NCMD} "runup runup"
@@ -994,7 +994,7 @@ RUN_COMMAND(){
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
#echo "Running command: ${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}"
${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}