voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update nodemgmt-scripts.sh

Browse Source
This commit is contained in:
David Schroeder
2022-12-17 11:23:32 -06:00
parent caff93d0f2
commit 25843a7742
1 changed files with 35 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
nodemgmt-scripts.sh
View File

@@ -138,6 +138,9 @@ STATUS(){
fi fi
nid=1 nid=1
dockers=${NTYPE}_DOCKER[@]
hosts=${NTYPE}_HOSTS[@]
var=${NTYPE}_HOSTS[@] var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
@@ -161,6 +164,7 @@ STATUS(){
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
fi fi
var2=${NTYPE}_SERVICES_CHECK[@] var2=${NTYPE}_SERVICES_CHECK[@]
for srvc in "${!var2}"; do for srvc in "${!var2}"; do
NOGOCHK=true; NOGOCHK=true;
@@ -242,64 +246,14 @@ STATUS(){
fi fi
fi fi
done done
else
if [ ! -f ${FOLDER}/${nip}.down ]; then if [[ -v ${NTYPE}_DOCKER ]]; then
touch ${FOLDER}/${nip}.down
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
touch ${FOLDER}/${nip}.errtime
fi
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# fi
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
# if [ $errtime -gt ${RENOTIFY} ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# fi
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
######### CHECK DOCKER STATUS ###############
dockers=${NTYPE}_DOCKER[@]
hosts=${NTYPE}_HOSTS[@]
if [[ -v ${NTYPE}_DOCKER ]]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${!hosts}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p` echo
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
DIVIDER false green echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}"
fi fi
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
fi
for docker in "${!dockers}"; do for docker in "${!dockers}"; do
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=30; spc='' c=0; cw=30; spc=''
@@ -328,25 +282,32 @@ STATUS(){
fi fi
fi fi
done done
else
if [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
touch ${FOLDER}/${nip}.errtime
fi
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
fi fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1` else
done if [ ! -f ${FOLDER}/${nip}.down ]; then
fi touch ${FOLDER}/${nip}.down
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
touch ${FOLDER}/${nip}.errtime
fi
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# fi
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
# if [ $errtime -gt ${RENOTIFY} ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# fi
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
done done
fi fi