voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-05-21 20:16:48 -05:00
parent 18e99f5264
commit cbc7c0d1ed
2 changed files with 28 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
inc/status.inc
View File

@@ -62,35 +62,35 @@ STATUS(){
fi fi
if [ "$(systemctl is-active ${srvc})" != "active" ]; then if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1 SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1
touch ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down touch ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
mv ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime mv ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
fi fi
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi fi
touch ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down touch ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down
fi fi
fi fi
else else
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}" echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi fi
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down ]; then
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
else else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down)
fi fi
rm -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.* rm -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.*
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
@@ -185,17 +185,17 @@ STATUS(){
fi fi
# echo "mysqlgo=${mysqlgo}" # echo "mysqlgo=${mysqlgo}"
if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then if [ ! -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
touch ${TMPFOLDER}/${nip}-${srvc}.down touch ${TMPFOLDER}/${nip}.${srvc}.down
# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY} ]; then
[ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}-${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime [ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}.${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime)
touch ${TMPFOLDER}/${nip}-${srvc}.down touch ${TMPFOLDER}/${nip}.${srvc}.down
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
@@ -207,8 +207,8 @@ STATUS(){
fi fi
else else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then if [ -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then
[ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) [ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down)
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
@@ -255,22 +255,22 @@ STATUS(){
# fi # fi
# # echo "mysqlgo=${mysqlgo}" # # echo "mysqlgo=${mysqlgo}"
# if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then # if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
# if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then # if [ ! -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then
# if [ "${ST_ACTION}" != "check" ]; then # if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" # echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
# fi # fi
# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
# touch ${TMPFOLDER}/${nip}-${srvc}.down # touch ${TMPFOLDER}/${nip}.${srvc}.down
# echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
# #
# else # else
# errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) # errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down)
# if [ $errtime -gt ${RENOTIFY} ]; then # if [ $errtime -gt ${RENOTIFY} ]; then
# if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then # if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then
# mv ${TMPFOLDER}/${nip}-${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime # mv ${TMPFOLDER}/${nip}.${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime
# fi # fi
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime)
# touch ${TMPFOLDER}/${nip}-${srvc}.down # touch ${TMPFOLDER}/${nip}.${srvc}.down
# if [ "${srvc}" = "mysql" ]; then # if [ "${srvc}" = "mysql" ]; then
# ${NCMD} systemctl restart ${srvc} # ${NCMD} systemctl restart ${srvc}
# fi # fi
@@ -283,22 +283,22 @@ STATUS(){
# if [ "${ST_ACTION}" != "check" ]; then # if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Green]}Running${idsCL[Default]}" # echo -e "${idsCL[Green]}Running${idsCL[Default]}"
# fi # fi
# if [ -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then # if [ -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then
# if [ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then # if [ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime)
# else # else
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down)
# fi # fi
# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
# echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
# #
# rm -f ${TMPFOLDER}/${nip}-${srvc}.down # rm -f ${TMPFOLDER}/${nip}.${srvc}.down
# rm -f ${TMPFOLDER}/${nip}-${srvc}.errtime # rm -f ${TMPFOLDER}/${nip}-${srvc}.errtime
# #
# fi # fi
# # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then # # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
# if [ "${srvc}" == "gitea" ]; then # if [ "${srvc}" == "gitea" ]; then
# rm -f ${FOLDER}/*-${srvc}.down # rm -f ${FOLDER}/*.${srvc}.down
# rm -f ${FOLDER}/*-${srvc}.errtime # rm -f ${FOLDER}/*-${srvc}.errtime
# fi # fi
# fi # fi

10
nodemgmt-scripts.sh
View File

@@ -1074,14 +1074,14 @@ DOWNLIST(){
for downitem in ${DOWN_LIST}; do for downitem in ${DOWN_LIST}; do
ditem=${downitem##*/} ditem=${downitem##*/}
ditem=${ditem//.down} ditem=${ditem//.down}
host=${ditem%%-*} host=${ditem%%.*}
item=${ditem#*-} item=${ditem#*.}
if [[ $ditem == *"-"* ]] && [ "${NM_SERVICES[${item}]}" != "" ]; then if [[ $ditem == *"."* ]] && [ "${NM_SERVICES[${item}]}" != "" ]; then
itemname=${NM_SERVICES[${ditem#*-}]} itemname=${NM_SERVICES[${ditem#*-}]}
elif [[ $ditem == *"-"* ]] && [ "${NM_DOCKERS[${item}]}" != "" ]; then elif [[ $ditem == *"."* ]] && [ "${NM_DOCKERS[${item}]}" != "" ]; then
itemname="${NM_DOCKERS[${ditem#*-}]} (docker)" itemname="${NM_DOCKERS[${ditem#*-}]} (docker)"
elif [[ $ditem == *"-"* ]] && [ "${REPL_DESC[${item}]}" != "" ]; then elif [[ $ditem == *"."* ]] && [ "${REPL_DESC[${item}]}" != "" ]; then
itemname="${REPL_DESC[${item}]} (replication)" itemname="${REPL_DESC[${item}]} (replication)"
else else
itemname='HOST DOWN'; itemname='HOST DOWN';