voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update nodemgmt-scripts.sh

Browse Source
This commit is contained in:
David Schroeder
2019-02-07 22:18:10 -06:00
parent 103321aea2
commit 0083753218
1 changed files with 43 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
nodemgmt-scripts.sh
View File

@@ -207,12 +207,11 @@ STATUS-CHECK(){
else NCMD="ssh root@${nip}"
fi
for srvc in "${NODE_SERVICES[@]}"; do
echo "${nip} - ${srvc}"
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
# if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
touch ${FOLDER}/${nip}-${srvc}.down
fi
# fi
elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
rm -f ${FOLDER}/${nip}-${srvc}.down
@@ -225,47 +224,47 @@ STATUS-CHECK(){
nid=`expr $nid + 1`
done
##REPLICATION CHECK
nid=1
for nip in "${NODE_HOSTS[@]}"; do
if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
rm -f ${FOLDER}/${nip}.down
echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
fi
for rcheck in "${REPL_CHECK[@]}"; do
timeout=`date --date='20 seconds' +%s`
checked=false
until [ "${checked}" = "" ]; do
if [ "${NHCMD}" = "" ]; then
checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
else
checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
fi
if [ "`date +%s`" -gt "$timeout" ]; then
timeout=true
break
fi
done
if [ "${timeout}" != "true" ]; then
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
rm -f ${FOLDER}/${nip}-${rcheck}.down
echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
fi
elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down
echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
fi
done
elif [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
fi
fi
nid=`expr $nid + 1`
done
# nid=1
# for nip in "${NODE_HOSTS[@]}"; do
# if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
# if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
# checkhost=$(CHECK_HOST ${nip})
# if [ "${checkhost}" != "false" ]; then
# if [ -f ${FOLDER}/${nip}.down ]; then
# rm -f ${FOLDER}/${nip}.down
# echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
# fi
# for rcheck in "${REPL_CHECK[@]}"; do
# timeout=`date --date='20 seconds' +%s`
# checked=false
# until [ "${checked}" = "" ]; do
# if [ "${NHCMD}" = "" ]; then
# checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
# else
# checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
# fi
# if [ "`date +%s`" -gt "$timeout" ]; then
# timeout=true
# break
# fi
# done
# if [ "${timeout}" != "true" ]; then
# if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
# rm -f ${FOLDER}/${nip}-${rcheck}.down
# echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
# fi
# elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
# touch ${FOLDER}/${nip}-${rcheck}.down
# echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
# fi
# done
# elif [ ! -f ${FOLDER}/${nip}.down ]; then
# touch ${FOLDER}/${nip}.down
# echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
# fi
# fi
# nid=`expr $nid + 1`
# done
fi
fi
end=`date +%s`