voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

simplifi-ntypes

Browse Source
This commit is contained in:
David Schroeder
2019-03-26 20:30:29 -05:00
parent 703afd7e15
commit 08c9fb6c7d
2 changed files with 120 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
defaults.inc
View File

@@ -1,13 +1,14 @@
#!/usr/bin/env bash
VERS='3.03252019'
NODEHOSTS=10.5.10.51,10.5.10.52,10.5.10.53
NODETYPES=MYSQL,LB,WEB
MYSQLHOSTS=10.5.10.51,10.5.10.52,10.5.10.53
WEBHOSTS=10.5.10.121,10.5.10.122,10.5.10.123
LBHOSTS=10.10.10.81,10.10.10.82,10.10.10.83
PRIMARYHOST=10.5.10.51
MYSQLMANAGER=10.5.10.50
NODESERVICES=mysql,nginx,gitea,powerdns-admin,haproxy,keepalived,maxscale,postgresql
NODESERVICES_CHECK=mysql
MYSQLSERVICES_CHECK=mysql
WEBSERVICES_CHECK=nginx,gitea,keepalived
LBSERVICES_CHECK=haproxy,keepalived
noheader=' service status-check nightlyrenew backup report check checkcerts gitea '
@@ -19,16 +20,22 @@ RENOTIFY=900
declare -i errtime
IFS=,
NODE_HOSTS=(${NODEHOSTS})
NODE_TYPES=(${NODETYPES})
MYSQL_HOSTS=(${MYSQLHOSTS})
WEB_HOSTS=(${WEBHOSTS})
LB_HOSTS=(${LBHOSTS})
NODE_SERVICES=(${NODESERVICES})
NODE_SERVICES_CHECK=(${NODESERVICES_CHECK})
MYSQL_SERVICES_CHECK=(${MYSQLSERVICES_CHECK})
WEB_SERVICES_CHECK=(${WEBSERVICES_CHECK})
LB_SERVICES_CHECK=(${LBSERVICES_CHECK})
LOCAL_SERVICES=(${LOCAL_SERVICES})
unset IFS
declare -A NM_NODETYPES
NM_NODETYPES['MYSQL']='MySQL'
NM_NODETYPES['LB']='LoadBalance'
NM_NODETYPES['WEB']='Webserver'
declare -A NM_SERVICES
NM_SERVICES['mysql']='MySQL'
NM_SERVICES['postgresql']='PostgreSQL'
@@ -63,7 +70,7 @@ REPL_CHECKS['www']='/var/www'
REPL_CHECKS['git']='/var/lib/gitea'
REPL_CHECKS['gitssh']='/home/git'
NODE_REPL_CHECK=( )
MYSQL_REPL_CHECK=( )
WEB_REPL_CHECK=(nginx ssl php www git gitssh)
LB_REPL_CHECK=( )

488
nodemgmt-scripts.sh
View File

@@ -37,15 +37,15 @@ STATUS(){
echo
fi
# if [ ! -z ${LOCAL_SERVICES+x} ]; then
NHCMD="ssh root@${NODE_HOSTS[0]}"
NHCMD="ssh root@${MYSQL_HOSTS[0]}"
LBHCMD="ssh root@${LB_HOSTS[0]}"
WHCMD="ssh root@${WEB_HOSTS[0]}"
# echo "HERE: '${NHCMD}' '${WHCMD}'"
# else
# NHCMD=''
# fi
if [ ! -z ${NODE_REPL_CHECK+x} ]; then
for rcheck in "${NODE_REPL_CHECK[@]}"; do
if [ ! -z ${MYSQL_REPL_CHECK+x} ]; then
for rcheck in "${MYSQL_REPL_CHECK[@]}"; do
${NHCMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${NHCMD}" = "" ]; then
@@ -86,6 +86,7 @@ STATUS(){
fi
fi
########################
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then
if [ ! -z ${LOCAL_SERVICES+x} ]; then
if [ "${ST_ACTION}" != "check" ]; then
@@ -143,379 +144,106 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi
########################
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} MySQL-Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${NODE_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
for NTYPE in "${NODE_TYPES[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}MySQL-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "MySQL-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
nid=1
var="${NTYPE}_HOSTS"
for nip in "${var[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
for srvc in "${NODE_SERVICES_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "MySQL-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi
var2="${NTYPE}_SERVICES_CHECK"
for srvc in "${var2[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "MySQL-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "MySQL-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "MySQL-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}.down
fi
touch ${FOLDER}/${nip}.down
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
########################
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Load-Balance-Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${LB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}LB-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi
for srvc in "${LB_SERVICES_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
fi
touch ${FOLDER}/${nip}.down
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
########################
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Webserver-Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}Webserver-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Webserver-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi
for srvc in "${WEB_SERVICES_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Webserver-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
fi
touch ${FOLDER}/${nip}.down
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
nid=1
for nip in "${LB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}LB-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi
for srvc in "${LB_SERVICES_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
fi
touch ${FOLDER}/${nip}.down
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
done
fi
########################
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then
@@ -524,12 +252,12 @@ STATUS(){
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
if [ ! -z ${NODE_REPL_CHECK+x} ]; then
if [ ! -z ${MYSQL_REPL_CHECK+x} ]; then
nid=1
for nip in "${NODE_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${NODE_HOSTS[0]}" ]; then
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${MYSQL_HOSTS[0]}" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsCL[LightCyan]}MySQL-Node1 (${NODE_HOSTS[0]}) <--> ${idsST[Bold]}MySQL-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
echo -e " ${idsCL[LightCyan]}MySQL-Node1 (${MYSQL_HOSTS[0]}) <--> ${idsST[Bold]}MySQL-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
checkhost=$(CHECK_HOST ${nip})
@@ -538,7 +266,7 @@ STATUS(){
rm -f ${FOLDER}/${nip}.down
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} is back UP!" | mail -s "MySQL-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi
for rcheck in "${NODE_REPL_CHECK[@]}"; do
for rcheck in "${MYSQL_REPL_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#REPL_DESC[${rcheck}]}
@@ -689,7 +417,7 @@ NEWCERT(){
echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}"
echo ""
for nip in "${NODE_HOSTS[@]}"; do
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
`ssh root@${nip} service nginx stop`
fi
@@ -712,7 +440,7 @@ NEWCERT(){
timeout=`date --date='2 minutes' +%s`
until [ "${checked}" = "" ]; do
tchecked=''
for nip in "${NODE_HOSTS[@]}"; do
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
else NCMD="ssh root@${nip}"
fi
@@ -739,7 +467,7 @@ NEWCERT(){
echo ""
echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}"
fi
for nip in "${NODE_HOSTS[@]}"; do
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
`ssh root@${nip} service nginx start`
fi
@@ -896,7 +624,7 @@ LISTCERTS(){
done
unset IFS
if [ ! -z ${LOCAL_SERVICES+x} ]; then
NCMD="ssh root@${NODE_HOSTS[0]}"
NCMD="ssh root@${MYSQL_HOSTS[0]}"
${NCMD} rm -f /tmp/ssllist
${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist
else
@@ -1280,7 +1008,7 @@ NEWSITE(){
fi
echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE}
for nip in "${NODE_HOSTS[@]}"; do
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
else NCMD="ssh root@${nip}"
fi
@@ -1338,7 +1066,7 @@ NEWSITE(){
timeout=`date --date='2 minutes' +%s`
until [ "${checked}" = "" ]; do
tchecked=''
for nip in "${NODE_HOSTS[@]}"; do
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
else NCMD="ssh root@${nip}"
fi
@@ -1398,7 +1126,7 @@ SET-PERMISSIONS(){
fi
if [ "${NEW_SITE}" = "xxx" ]; then
nid=1
for nip in "${NODE_HOSTS[@]}"
for nip in "${MYSQL_HOSTS[@]}"
do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
@@ -1556,9 +1284,9 @@ SERVICE(){
echo -e "${idsCL[LightGreen]}[[${NM_SERVICES[${1}]} ${NM_SRVCOPTS[${2}]}ing]]${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
if [[ "${NODESERVICES_CHECK}" = *"${1}"* ]]; then
if [[ "${MYSQLSERVICES_CHECK}" = *"${1}"* ]]; then
nid=1
for nip in "${NODE_HOSTS[@]}"; do
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
@@ -1640,9 +1368,9 @@ SERVICE(){
HAPROXY-CONFIG() {
echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}"
DIVIDER false yellow
if [[ "${NODESERVICES_CHECK}" = *"haproxy"* ]]; then
if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then
nid=1
for nip in "${NODE_HOSTS[@]}"; do
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
@@ -1707,9 +1435,9 @@ HAPROXY-CONFIG() {
KEEPALIVE-CONFIG() {
echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"
DIVIDER false yellow
if [[ "${NODESERVICES_CHECK}" = *"keepalive"* ]]; then
if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then
nid=1
for nip in "${NODE_HOSTS[@]}"; do
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
@@ -1767,7 +1495,7 @@ NODEUPDATE() {
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
echo
nid=1
for nip in "${NODE_HOSTS[@]}"; do
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD='ssh root@localhost'