voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

simplifi-ntypes

Browse Source
This commit is contained in:
David Schroeder
2019-03-26 20:30:29 -05:00
parent 703afd7e15
commit 08c9fb6c7d
2 changed files with 120 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
defaults.inc
View File

@@ -1,13 +1,14 @@
#!/usr/bin/env bash #!/usr/bin/env bash
VERS='3.03252019' VERS='3.03252019'
NODEHOSTS=10.5.10.51,10.5.10.52,10.5.10.53 NODETYPES=MYSQL,LB,WEB
MYSQLHOSTS=10.5.10.51,10.5.10.52,10.5.10.53
WEBHOSTS=10.5.10.121,10.5.10.122,10.5.10.123 WEBHOSTS=10.5.10.121,10.5.10.122,10.5.10.123
LBHOSTS=10.10.10.81,10.10.10.82,10.10.10.83 LBHOSTS=10.10.10.81,10.10.10.82,10.10.10.83
PRIMARYHOST=10.5.10.51 PRIMARYHOST=10.5.10.51
MYSQLMANAGER=10.5.10.50 MYSQLMANAGER=10.5.10.50
NODESERVICES=mysql,nginx,gitea,powerdns-admin,haproxy,keepalived,maxscale,postgresql NODESERVICES=mysql,nginx,gitea,powerdns-admin,haproxy,keepalived,maxscale,postgresql
NODESERVICES_CHECK=mysql MYSQLSERVICES_CHECK=mysql
WEBSERVICES_CHECK=nginx,gitea,keepalived WEBSERVICES_CHECK=nginx,gitea,keepalived
LBSERVICES_CHECK=haproxy,keepalived LBSERVICES_CHECK=haproxy,keepalived
noheader=' service status-check nightlyrenew backup report check checkcerts gitea ' noheader=' service status-check nightlyrenew backup report check checkcerts gitea '
@@ -19,16 +20,22 @@ RENOTIFY=900
declare -i errtime declare -i errtime
IFS=, IFS=,
NODE_HOSTS=(${NODEHOSTS}) NODE_TYPES=(${NODETYPES})
MYSQL_HOSTS=(${MYSQLHOSTS})
WEB_HOSTS=(${WEBHOSTS}) WEB_HOSTS=(${WEBHOSTS})
LB_HOSTS=(${LBHOSTS}) LB_HOSTS=(${LBHOSTS})
NODE_SERVICES=(${NODESERVICES}) NODE_SERVICES=(${NODESERVICES})
NODE_SERVICES_CHECK=(${NODESERVICES_CHECK}) MYSQL_SERVICES_CHECK=(${MYSQLSERVICES_CHECK})
WEB_SERVICES_CHECK=(${WEBSERVICES_CHECK}) WEB_SERVICES_CHECK=(${WEBSERVICES_CHECK})
LB_SERVICES_CHECK=(${LBSERVICES_CHECK}) LB_SERVICES_CHECK=(${LBSERVICES_CHECK})
LOCAL_SERVICES=(${LOCAL_SERVICES}) LOCAL_SERVICES=(${LOCAL_SERVICES})
unset IFS unset IFS
declare -A NM_NODETYPES
NM_NODETYPES['MYSQL']='MySQL'
NM_NODETYPES['LB']='LoadBalance'
NM_NODETYPES['WEB']='Webserver'
declare -A NM_SERVICES declare -A NM_SERVICES
NM_SERVICES['mysql']='MySQL' NM_SERVICES['mysql']='MySQL'
NM_SERVICES['postgresql']='PostgreSQL' NM_SERVICES['postgresql']='PostgreSQL'
@@ -63,7 +70,7 @@ REPL_CHECKS['www']='/var/www'
REPL_CHECKS['git']='/var/lib/gitea' REPL_CHECKS['git']='/var/lib/gitea'
REPL_CHECKS['gitssh']='/home/git' REPL_CHECKS['gitssh']='/home/git'
NODE_REPL_CHECK=( ) MYSQL_REPL_CHECK=( )
WEB_REPL_CHECK=(nginx ssl php www git gitssh) WEB_REPL_CHECK=(nginx ssl php www git gitssh)
LB_REPL_CHECK=( ) LB_REPL_CHECK=( )

488
nodemgmt-scripts.sh
View File

@@ -37,15 +37,15 @@ STATUS(){
echo echo
fi fi
# if [ ! -z ${LOCAL_SERVICES+x} ]; then # if [ ! -z ${LOCAL_SERVICES+x} ]; then
NHCMD="ssh root@${NODE_HOSTS[0]}" NHCMD="ssh root@${MYSQL_HOSTS[0]}"
LBHCMD="ssh root@${LB_HOSTS[0]}" LBHCMD="ssh root@${LB_HOSTS[0]}"
WHCMD="ssh root@${WEB_HOSTS[0]}" WHCMD="ssh root@${WEB_HOSTS[0]}"
# echo "HERE: '${NHCMD}' '${WHCMD}'" # echo "HERE: '${NHCMD}' '${WHCMD}'"
# else # else
# NHCMD='' # NHCMD=''
# fi # fi
if [ ! -z ${NODE_REPL_CHECK+x} ]; then if [ ! -z ${MYSQL_REPL_CHECK+x} ]; then
for rcheck in "${NODE_REPL_CHECK[@]}"; do for rcheck in "${MYSQL_REPL_CHECK[@]}"; do
${NHCMD} rm -f ${FOLDER}/test.repl ${NHCMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S` daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${NHCMD}" = "" ]; then if [ "${NHCMD}" = "" ]; then
@@ -86,6 +86,7 @@ STATUS(){
fi fi
fi fi
######################## ########################
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then
if [ ! -z ${LOCAL_SERVICES+x} ]; then if [ ! -z ${LOCAL_SERVICES+x} ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
@@ -143,379 +144,106 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then echo; fi if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi fi
######################## ########################
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER for NTYPE in "${NODE_TYPES[@]}"; do
echo -e "${idsCL[Yellow]} MySQL-Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${NODE_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p` echo -e "${idsST[Bold]}"; DIVIDER
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}MySQL-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
DIVIDER false green DIVIDER; echo -e "${idsST[Reset]}"
fi fi
if [ "${NCMD}" != "" ]; then nid=1
checkhost=$(CHECK_HOST ${nip}) var="${NTYPE}_HOSTS"
fi for nip in "${var[@]}"; do
if [ "${checkhost}" != "false" ]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
if [ -f ${FOLDER}/${nip}.down ]; then else NCMD="ssh root@${nip}"; LH=''
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "MySQL-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi fi
for srvc in "${NODE_SERVICES_CHECK[@]}"; do if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "check" ]; then uptime=`${NCMD} uptime -p`
c=0; cw=22; spc='' echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
spc1=${cw}-${#NM_SERVICES[${srvc}]} DIVIDER false green
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done fi
echo -en " ${NM_SERVICES[${srvc}]}$spc: " if [ "${NCMD}" != "" ]; then
fi checkhost=$(CHECK_HOST ${nip})
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then if [ "${checkhost}" != "false" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then if [ -f ${FOLDER}/${nip}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ -f ${FOLDER}/${nip}.errtime ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "MySQL-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
else else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then fi
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) var2="${NTYPE}_SERVICES_CHECK"
for srvc in "${var2[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "MySQL-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi fi
fi fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else else
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}" echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "MySQL-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi fi
fi fi
done touch ${FOLDER}/${nip}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "MySQL-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
fi fi
touch ${FOLDER}/${nip}.down if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi nid=`expr $nid + 1`
if [ "${ST_ACTION}" != "check" ]; then echo; fi done
nid=`expr $nid + 1`
done
########################
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Load-Balance-Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${LB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}LB-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi
for srvc in "${LB_SERVICES_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
fi
touch ${FOLDER}/${nip}.down
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
########################
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Webserver-Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}Webserver-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Webserver-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi
for srvc in "${WEB_SERVICES_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Webserver-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
fi
touch ${FOLDER}/${nip}.down
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
nid=1
for nip in "${LB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}LB-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi
for srvc in "${LB_SERVICES_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "Webserver-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
fi
touch ${FOLDER}/${nip}.down
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done done
fi fi
######################## ########################
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then
@@ -524,12 +252,12 @@ STATUS(){
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}" DIVIDER; echo -e "${idsST[Reset]}"
fi fi
if [ ! -z ${NODE_REPL_CHECK+x} ]; then if [ ! -z ${MYSQL_REPL_CHECK+x} ]; then
nid=1 nid=1
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${NODE_HOSTS[0]}" ]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${MYSQL_HOSTS[0]}" ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsCL[LightCyan]}MySQL-Node1 (${NODE_HOSTS[0]}) <--> ${idsST[Bold]}MySQL-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" echo -e " ${idsCL[LightCyan]}MySQL-Node1 (${MYSQL_HOSTS[0]}) <--> ${idsST[Bold]}MySQL-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green DIVIDER false green
fi fi
checkhost=$(CHECK_HOST ${nip}) checkhost=$(CHECK_HOST ${nip})
@@ -538,7 +266,7 @@ STATUS(){
rm -f ${FOLDER}/${nip}.down rm -f ${FOLDER}/${nip}.down
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} is back UP!" | mail -s "MySQL-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "MySQL-Node${nid} is back UP!" | mail -s "MySQL-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi fi
for rcheck in "${NODE_REPL_CHECK[@]}"; do for rcheck in "${MYSQL_REPL_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc='' c=0; cw=22; spc=''
spc1=${cw}-${#REPL_DESC[${rcheck}]} spc1=${cw}-${#REPL_DESC[${rcheck}]}
@@ -689,7 +417,7 @@ NEWCERT(){
echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}" echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}"
echo "" echo ""
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
`ssh root@${nip} service nginx stop` `ssh root@${nip} service nginx stop`
fi fi
@@ -712,7 +440,7 @@ NEWCERT(){
timeout=`date --date='2 minutes' +%s` timeout=`date --date='2 minutes' +%s`
until [ "${checked}" = "" ]; do until [ "${checked}" = "" ]; do
tchecked='' tchecked=''
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
else NCMD="ssh root@${nip}" else NCMD="ssh root@${nip}"
fi fi
@@ -739,7 +467,7 @@ NEWCERT(){
echo "" echo ""
echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}" echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}"
fi fi
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
`ssh root@${nip} service nginx start` `ssh root@${nip} service nginx start`
fi fi
@@ -896,7 +624,7 @@ LISTCERTS(){
done done
unset IFS unset IFS
if [ ! -z ${LOCAL_SERVICES+x} ]; then if [ ! -z ${LOCAL_SERVICES+x} ]; then
NCMD="ssh root@${NODE_HOSTS[0]}" NCMD="ssh root@${MYSQL_HOSTS[0]}"
${NCMD} rm -f /tmp/ssllist ${NCMD} rm -f /tmp/ssllist
${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist ${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist
else else
@@ -1280,7 +1008,7 @@ NEWSITE(){
fi fi
echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE} echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE}
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
else NCMD="ssh root@${nip}" else NCMD="ssh root@${nip}"
fi fi
@@ -1338,7 +1066,7 @@ NEWSITE(){
timeout=`date --date='2 minutes' +%s` timeout=`date --date='2 minutes' +%s`
until [ "${checked}" = "" ]; do until [ "${checked}" = "" ]; do
tchecked='' tchecked=''
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
else NCMD="ssh root@${nip}" else NCMD="ssh root@${nip}"
fi fi
@@ -1398,7 +1126,7 @@ SET-PERMISSIONS(){
fi fi
if [ "${NEW_SITE}" = "xxx" ]; then if [ "${NEW_SITE}" = "xxx" ]; then
nid=1 nid=1
for nip in "${NODE_HOSTS[@]}" for nip in "${MYSQL_HOSTS[@]}"
do do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost ' nip='localhost '
@@ -1556,9 +1284,9 @@ SERVICE(){
echo -e "${idsCL[LightGreen]}[[${NM_SERVICES[${1}]} ${NM_SRVCOPTS[${2}]}ing]]${idsCL[Default]}" echo -e "${idsCL[LightGreen]}[[${NM_SERVICES[${1}]} ${NM_SRVCOPTS[${2}]}ing]]${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}" echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
if [[ "${NODESERVICES_CHECK}" = *"${1}"* ]]; then if [[ "${MYSQLSERVICES_CHECK}" = *"${1}"* ]]; then
nid=1 nid=1
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost ' nip='localhost '
NCMD='' NCMD=''
@@ -1640,9 +1368,9 @@ SERVICE(){
HAPROXY-CONFIG() { HAPROXY-CONFIG() {
echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}" echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}"
DIVIDER false yellow DIVIDER false yellow
if [[ "${NODESERVICES_CHECK}" = *"haproxy"* ]]; then if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then
nid=1 nid=1
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost ' nip='localhost '
NCMD='' NCMD=''
@@ -1707,9 +1435,9 @@ HAPROXY-CONFIG() {
KEEPALIVE-CONFIG() { KEEPALIVE-CONFIG() {
echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}" echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"
DIVIDER false yellow DIVIDER false yellow
if [[ "${NODESERVICES_CHECK}" = *"keepalive"* ]]; then if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then
nid=1 nid=1
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost ' nip='localhost '
NCMD='' NCMD=''
@@ -1767,7 +1495,7 @@ NODEUPDATE() {
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}" echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
echo echo
nid=1 nid=1
for nip in "${NODE_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost ' nip='localhost '
NCMD='ssh root@localhost' NCMD='ssh root@localhost'