voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update nodemgmt-scripts.sh

Browse Source
This commit is contained in:
David Schroeder
2019-02-08 00:03:45 -06:00
parent e8210b467c
commit 5f4ba331d5
1 changed files with 66 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
nodemgmt-scripts.sh
View File

@@ -27,6 +27,7 @@ STATUS(){
idsST=('')
fi
declare -i cw; declare -i spc1; declare -i c
########################
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}"
@@ -51,7 +52,7 @@ STATUS(){
sleep 5
fi
fi
########################
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
@@ -65,7 +66,6 @@ STATUS(){
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
DIVIDER false green
fi
for srvc in "${LOCAL_SERVICES[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
@@ -73,20 +73,25 @@ STATUS(){
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(systemctl is-active ${srvc})" = "active" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${FOLDER}/localhost-${srvc}.down
else
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
touch ${FOLDER}/localhost-${srvc}.down
if [ ! -f ${FOLDER}/localhost-${srvc}.down ]; then
echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
touch ${FOLDER}/localhost-${srvc}.down
fi
elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
rm -f ${FOLDER}/localhost-${srvc}.down; rm -f ${FOLDER}/localhost-${srvc}.errtime
fi
done
echo
fi
########################
nid=1
for nip in "${NODE_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
@@ -101,7 +106,10 @@ STATUS(){
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
rm -f ${FOLDER}/${nip}.down
if [ -f ${FOLDER}/${nip}.down ]; then
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
fi
for srvc in "${NODE_SERVICES[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
@@ -109,33 +117,35 @@ STATUS(){
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" = "active" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
elif [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${srvc}.down
else
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
done
else
@@ -148,7 +158,7 @@ STATUS(){
nid=`expr $nid + 1`
done
fi
########################
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
@@ -165,7 +175,10 @@ STATUS(){
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
rm -f ${FOLDER}/${nip}.down
if [ -f ${FOLDER}/${nip}.down ]; then
rm -f ${FOLDER}/${nip}.down
echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
fi
for rcheck in "${REPL_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
@@ -186,23 +199,33 @@ STATUS(){
break
fi
done
if [ "${timeout}" != "true" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
fi
rm -f ${FOLDER}/${nip}-${rcheck}.down
else
if [ "${timeout}" = "true" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${rcheck}.down
if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down
echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
rm -f ${FOLDER}/${nip}-${rcheck}.down
echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
fi
fi
done
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}.down
if [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi