voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update nodemgmt-scripts.sh

Browse Source
This commit is contained in:
David Schroeder
2023-01-01 17:55:41 -06:00
parent fc311ab128
commit ebdb7450f1
1 changed files with 15 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
nodemgmt-scripts.sh
View File

@@ -95,6 +95,7 @@ STATUS(){
fi
if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi
touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
@@ -122,6 +123,8 @@ STATUS(){
if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi
rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE}
fi
fi
done
@@ -180,7 +183,9 @@ STATUS(){
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
# if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${EMAIL_NOTICE}; fi
if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${EMAIL_NOTICE}; fi
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
fi
if [ "${ST_ACTION}" != "dockers" ]; then
@@ -218,13 +223,13 @@ STATUS(){
# echo "mysqlgo=${mysqlgo}"
if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
touch ${FOLDER}/${nip}-${srvc}.down
# fi
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
@@ -253,6 +258,7 @@ STATUS(){
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
rm -f ${FOLDER}/*-${srvc}.down
@@ -300,6 +306,8 @@ STATUS(){
if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
rm -f ${FOLDER}/${nip}-${docker}.down
rm -f ${FOLDER}/${nip}-${docker}.errtime
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
@@ -307,6 +315,7 @@ STATUS(){
if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
touch ${FOLDER}/${nip}-${docker}.down
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
@@ -344,6 +353,7 @@ STATUS(){
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node is down!${idsCL[Default]}"
fi
[ "${EMAIL_NOTICE}" != "" ] && echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
@@ -457,6 +467,7 @@ STATUS(){
if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down
if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\t$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi
else
if [ "${ST_ACTION}" != "check" ]; then
@@ -465,6 +476,7 @@ STATUS(){
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
rm -f ${FOLDER}/${nip}-${rcheck}.down
if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\t$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi
fi