voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update nodemgmt-scripts.sh

Browse Source
This commit is contained in:
David Schroeder
2019-03-25 12:27:33 -05:00
parent d4d7850b81
commit f86efd9a29
1 changed files with 105 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
nodemgmt-scripts.sh
View File

@@ -38,6 +38,7 @@ STATUS(){
fi
# if [ ! -z ${LOCAL_SERVICES+x} ]; then
NHCMD="ssh root@${NODE_HOSTS[0]}"
LBHCMD="ssh root@${LB_HOSTS[0]}"
WHCMD="ssh root@${WEB_HOSTS[0]}"
# echo "HERE: '${NHCMD}' '${WHCMD}'"
# else
@@ -56,6 +57,19 @@ STATUS(){
fi
done
fi
if [ ! -z ${LB_REPL_CHECK+x} ]; then
for rcheck in "${LB_REPL_CHECK[@]}"; do
${LBHCMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${LBHCMD}" = "" ]; then
echo -e "LB-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
else
${LBHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
yes | ${LBHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
fi
done
fi
for rcheck in "${WEB_REPL_CHECK[@]}"; do
${WHCMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
@@ -225,6 +239,96 @@ STATUS(){
done
########################
nid=1
for nip in "${LB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}LB-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi
for srvc in "${LB_SERVICES_CHECK[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
fi
done
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
mv ${FOLDER}/${nip}.down ${FOLDER}/${nip}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node has been down for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "LB-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "LB-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
fi
touch ${FOLDER}/${nip}.down
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
########################
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
@@ -1694,7 +1798,7 @@ NODEUPDATE() {
nid=`expr $nid + 1`
done
echo ""
echo -e "${idsCL[Green]}All Nodes and Webservers have been Updated.${idsCL[Default]}"
echo -e "${idsCL[Green]}All Nodes have been Updated.${idsCL[Default]}"
echo ""
}