voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update nodemgmt-scripts.sh

Browse Source
This commit is contained in:
David Schroeder
2019-02-07 14:39:46 -06:00
parent 558f60d5f5
commit 136b4dee52
1 changed files with 60 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
nodemgmt-scripts.sh
View File

@@ -80,6 +80,7 @@ STATUS() {
DIVIDER false green DIVIDER false green
checkhost=$(CHECK_HOST ${nip}) checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then if [ "${checkhost}" != "false" ]; then
rm -f ${FOLDER}/${nip}.down
for srvc in "${NODE_SERVICES[@]}" for srvc in "${NODE_SERVICES[@]}"
do do
c=0; cw=22; spc='' c=0; cw=22; spc=''
@@ -96,6 +97,7 @@ STATUS() {
done done
else else
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
touch ${FOLDER}/${nip}.down
fi fi
echo echo
nid=`expr $nid + 1` nid=`expr $nid + 1`
@@ -112,6 +114,7 @@ STATUS() {
DIVIDER false green DIVIDER false green
checkhost=$(CHECK_HOST ${nip}) checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then if [ "${checkhost}" != "false" ]; then
rm -f ${FOLDER}/${nip}.down
for rcheck in "${REPL_CHECK[@]}"; do for rcheck in "${REPL_CHECK[@]}"; do
c=0; cw=22; spc='' c=0; cw=22; spc=''
spc1=${cw}-${#REPL_DESC[${rcheck}]} spc1=${cw}-${#REPL_DESC[${rcheck}]}
@@ -142,6 +145,7 @@ STATUS() {
done done
else else
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
touch ${FOLDER}/${nip}.down
fi fi
echo echo
fi fi
@@ -186,55 +190,72 @@ STATUS-CHECK(){
fi fi
if [ -z ${LOCAL_SERVICES+x} ] || [ "${2}" = "all" ]; then if [ -z ${LOCAL_SERVICES+x} ] || [ "${2}" = "all" ]; then
nid=1 nid=1
for nip in "${NODE_HOSTS[@]}" for nip in "${NODE_HOSTS[@]}"; do
do checkhost=$(CHECK_HOST ${nip})
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' if [ "${checkhost}" != "false" ]; then
else NCMD="ssh root@${nip}" if [ -f ${FOLDER}/${nip}.down ]; then
fi rm -f ${FOLDER}/${nip}.down
for srvc in "${NODE_SERVICES[@]}" echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
do
if [ $(${NCMD} pgrep ${srvc} | wc -l) -lt "1" ]; then
echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
touch ${FOLDER}/${nip}-${srvc}.down
elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
rm -f ${FOLDER}/${nip}-${srvc}.down
fi fi
done if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
else NCMD="ssh root@${nip}"
fi
for srvc in "${NODE_SERVICES[@]}"
do
if [ $(${NCMD} pgrep ${srvc} | wc -l) -lt "1" ]; then
echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
touch ${FOLDER}/${nip}-${srvc}.down
elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
rm -f ${FOLDER}/${nip}-${srvc}.down
fi
done
elif [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
fi
nid=`expr $nid + 1` nid=`expr $nid + 1`
done done
fi fi
##REPLICATION CHECK ##REPLICATION CHECK
nid=1 nid=1
for nip in "${NODE_HOSTS[@]}" for nip in "${NODE_HOSTS[@]}"; do
do
if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
for rcheck in "${REPL_CHECK[@]}"; do checkhost=$(CHECK_HOST ${nip})
timeout=`date --date='20 seconds' +%s` if [ "${checkhost}" != "false" ]; then
checked=false if [ -f ${FOLDER}/${nip}.down ]; then
until [ "${checked}" = "" ]; do rm -f ${FOLDER}/${nip}.down
if [ "${NCMD}" = "" ]; then echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` fi
else for rcheck in "${REPL_CHECK[@]}"; do
checked="`ssh root@10.5.10.51 \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" timeout=`date --date='20 seconds' +%s`
fi checked=false
if [ "`date +%s`" -gt "$timeout" ]; then until [ "${checked}" = "" ]; do
timeout=true if [ "${NCMD}" = "" ]; then
break checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
else
checked="`ssh root@10.5.10.51 \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
fi
if [ "`date +%s`" -gt "$timeout" ]; then
timeout=true
break
fi
done
if [ "${timeout}" != "true" ]; then
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
rm -f ${FOLDER}/${nip}-${rcheck}.down
echo "Replicated folder is backl up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
fi
elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down
echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
fi fi
done done
if [ "${timeout}" != "true" ]; then elif [ ! -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then touch ${FOLDER}/${nip}.down
rm -f ${FOLDER}/${nip}-${rcheck}.down echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
echo "Replicated folder is backl up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL} fi
fi
elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down
echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
fi
done
echo
fi fi
nid=`expr $nid + 1` nid=`expr $nid + 1`
done done