voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-05-01 22:06:25 -05:00
parent cc9271eae8
commit 6cb2e15fd4
3 changed files with 48 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
defaults.inc
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
VERS='4.6.23-04302023' VERS='4.6.24-04302023'
#NODETYPES=WEB,LB #NODETYPES=WEB,LB
NODETYPES=MYSQL,LB,WEB,NC,WPD,LPD,MM,PW NODETYPES=MYSQL,LB,WEB,NC,WPD,LPD,MM,PW

70
inc/status.inc
View File

@@ -150,11 +150,11 @@ STATUS(){
DIVIDER false green DIVIDER false green
fi fi
if [ -f ${FOLDER}/${nip}.down ]; then if [ -f ${TMPFOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then if [ -f ${TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
else else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
fi fi
rm -f ${FOLDER}/${nip}.* rm -f ${FOLDER}/${nip}.*
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
@@ -185,25 +185,25 @@ STATUS(){
fi fi
# echo "mysqlgo=${mysqlgo}" # echo "mysqlgo=${mysqlgo}"
if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
touch ${FOLDER}/${nip}-${srvc}.down touch ${TMPFOLDER}/${nip}-${srvc}.down
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY} ]; then
[ ! -f ${FOLDER}/${nip}-${srvc}.errtime ] && mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime [ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}-${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime)
touch ${FOLDER}/${nip}-${srvc}.down touch ${TMPFOLDER}/${nip}-${srvc}.down
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
fi fi
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi fi
else else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then if [ -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then
[ -f ${FOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) [ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down)
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
@@ -250,22 +250,22 @@ STATUS(){
# fi # fi
# # echo "mysqlgo=${mysqlgo}" # # echo "mysqlgo=${mysqlgo}"
# if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then # if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
# if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then # if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then
# if [ "${ST_ACTION}" != "check" ]; then # if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" # echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
# fi # fi
# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
# touch ${FOLDER}/${nip}-${srvc}.down # touch ${TMPFOLDER}/${nip}-${srvc}.down
# echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
# #
# else # else
# errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) # errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down)
# if [ $errtime -gt ${RENOTIFY} ]; then # if [ $errtime -gt ${RENOTIFY} ]; then
# if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then # if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then
# mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime # mv ${TMPFOLDER}/${nip}-${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime
# fi # fi
# toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime)
# touch ${FOLDER}/${nip}-${srvc}.down # touch ${TMPFOLDER}/${nip}-${srvc}.down
# if [ "${srvc}" = "mysql" ]; then # if [ "${srvc}" = "mysql" ]; then
# ${NCMD} systemctl restart ${srvc} # ${NCMD} systemctl restart ${srvc}
# fi # fi
@@ -278,17 +278,17 @@ STATUS(){
# if [ "${ST_ACTION}" != "check" ]; then # if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Green]}Running${idsCL[Default]}" # echo -e "${idsCL[Green]}Running${idsCL[Default]}"
# fi # fi
# if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then # if [ -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then
# if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then # if [ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then
# toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime)
# else # else
# toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down)
# fi # fi
# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
# echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
# #
# rm -f ${FOLDER}/${nip}-${srvc}.down # rm -f ${TMPFOLDER}/${nip}-${srvc}.down
# rm -f ${FOLDER}/${nip}-${srvc}.errtime # rm -f ${TMPFOLDER}/${nip}-${srvc}.errtime
# #
# fi # fi
# # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then # # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
@@ -371,23 +371,23 @@ STATUS(){
fi fi
else else
if [ ! -f ${FOLDER}/${nip}.down ]; then if [ ! -f ${TMPFOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down touch ${TMPFOLDER}/${nip}.down
if [ ! -f ${FOLDER}/${nip}.errtime ]; then if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
touch ${FOLDER}/${nip}.errtime touch ${TMPFOLDER}/${nip}.errtime
fi fi
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
fi fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1 SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE}
else else
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi fi
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then # if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE} # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}
# fi # fi
@@ -422,7 +422,7 @@ STATUS(){
var=${NTYPE}_HOSTS[@] var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@] var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do for rcheck in "${!var2}"; do
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1 REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
@@ -446,7 +446,7 @@ STATUS(){
echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green DIVIDER false green
fi fi
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@] var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
@@ -488,7 +488,7 @@ STATUS(){
else else
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}" [ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}"
[ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}" [ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}"
fi fi
fi fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi if [ "${ST_ACTION}" != "check" ]; then echo; fi

24
nodemgmt-scripts.sh
View File

@@ -40,14 +40,14 @@ CHECK_DOCKER_SERVICES(){
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green DIVIDER false green
fi fi
if [ -f ${FOLDER}/${nip}.down ]; then if [ -f ${TMPFOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then if [ -f ${TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
else else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
fi fi
rm -f ${FOLDER}/${nip}.down rm -f ${TMPFOLDER}/${nip}.down
rm -f ${FOLDER}/${nip}.errtime rm -f ${TMPFOLDER}/${nip}.errtime
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi fi
@@ -80,16 +80,16 @@ CHECK_DOCKER_SERVICES(){
fi fi
done done
else else
if [ ! -f ${FOLDER}/${nip}.down ]; then if [ ! -f ${TMPFOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down touch ${TMPFOLDER}/${nip}.down
if [ ! -f ${FOLDER}/${nip}.errtime ]; then if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
touch ${FOLDER}/${nip}.errtime touch ${TMPFOLDER}/${nip}.errtime
fi fi
else else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
fi fi
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi fi
fi fi