voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update status.inc

Browse Source
This commit is contained in:
David Schroeder
2023-05-23 21:25:48 -05:00
parent 9d11a6c872
commit 27b20c8d6a
1 changed files with 46 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
inc/status.inc
View File

@@ -74,37 +74,37 @@ STATUS(){
fi fi
if [ "$(systemctl is-active ${srvc})" != "active" ]; then if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down ]; then if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1 SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1
touch ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime ]; then if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
mv ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime mv ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
fi fi
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi fi
touch ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
fi fi
fi fi
else else
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}" echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi fi
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down ]; then if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime ]; then if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
else else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.down) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
fi fi
rm -f ${TMPFOLDER}/${NODE_HOSTNAME}.${srvc}.* rm -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE} echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE}
@@ -197,17 +197,17 @@ STATUS(){
fi fi
# echo "mysqlgo=${mysqlgo}" # echo "mysqlgo=${mysqlgo}"
if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
touch ${TMPFOLDER}/${nip}.${srvc}.down touch ${TMPFOLDER}/${nip}~${srvc}.down
# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY} ]; then
[ ! -f ${TMPFOLDER}/${nip}.${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}.${srvc}.down ${TMPFOLDER}/${nip}.${srvc}.errtime [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
touch ${TMPFOLDER}/${nip}.${srvc}.down touch ${TMPFOLDER}/${nip}~${srvc}.down
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
@@ -219,17 +219,17 @@ STATUS(){
fi fi
else else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${TMPFOLDER}/${nip}.${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down) [ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
rm -f ${TMPFOLDER}/${nip}.${srvc}.* rm -f ${TMPFOLDER}/${nip}~${srvc}.*
fi fi
# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
if [ "${srvc}" == "gitea" ]; then if [ "${srvc}" == "gitea" ]; then
rm -f ${FOLDER}/*.${srvc}.* rm -f ${FOLDER}/*~${srvc}.*
fi fi
fi fi
[ "${mysqlgo}" = "true" ] && unset mysqlgo [ "${mysqlgo}" = "true" ] && unset mysqlgo
@@ -267,22 +267,22 @@ STATUS(){
# fi # fi
# # echo "mysqlgo=${mysqlgo}" # # echo "mysqlgo=${mysqlgo}"
# if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then # if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
# if [ ! -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then # if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ "${ST_ACTION}" != "check" ]; then # if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" # echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
# fi # fi
# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
# touch ${TMPFOLDER}/${nip}.${srvc}.down # touch ${TMPFOLDER}/${nip}~${srvc}.down
# echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
# #
# else # else
# errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down) # errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
# if [ $errtime -gt ${RENOTIFY} ]; then # if [ $errtime -gt ${RENOTIFY} ]; then
# if [ ! -f ${TMPFOLDER}/${nip}.${srvc}.errtime ]; then # if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# mv ${TMPFOLDER}/${nip}.${srvc}.down ${TMPFOLDER}/${nip}.${srvc}.errtime # mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime
# fi # fi
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.errtime) # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
# touch ${TMPFOLDER}/${nip}.${srvc}.down # touch ${TMPFOLDER}/${nip}~${srvc}.down
# if [ "${srvc}" = "mysql" ]; then # if [ "${srvc}" = "mysql" ]; then
# ${NCMD} systemctl restart ${srvc} # ${NCMD} systemctl restart ${srvc}
# fi # fi
@@ -295,23 +295,23 @@ STATUS(){
# if [ "${ST_ACTION}" != "check" ]; then # if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Green]}Running${idsCL[Default]}" # echo -e "${idsCL[Green]}Running${idsCL[Default]}"
# fi # fi
# if [ -f ${TMPFOLDER}/${nip}.${srvc}.down ]; then # if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ -f ${TMPFOLDER}/${nip}.${srvc}.errtime ]; then # if [ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.errtime) # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
# else # else
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.${srvc}.down) # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
# fi # fi
# SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
# echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
# #
# rm -f ${TMPFOLDER}/${nip}.${srvc}.down # rm -f ${TMPFOLDER}/${nip}~${srvc}.down
# rm -f ${TMPFOLDER}/${nip}.${srvc}.errtime # rm -f ${TMPFOLDER}/${nip}~${srvc}.errtime
# #
# fi # fi
# # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then # # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
# if [ "${srvc}" == "gitea" ]; then # if [ "${srvc}" == "gitea" ]; then
# rm -f ${FOLDER}/*.${srvc}.down # rm -f ${FOLDER}/*~${srvc}.down
# rm -f ${FOLDER}/*.${srvc}.errtime # rm -f ${FOLDER}/*~${srvc}.errtime
# fi # fi
# fi # fi
# [ "${mysqlgo}" = "true" ] && unset mysqlgo # [ "${mysqlgo}" = "true" ] && unset mysqlgo
@@ -351,29 +351,28 @@ STATUS(){
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed" SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed"
rm -f ${TMPFOLDER}/${nip}-${docker}.down rm -f ${TMPFOLDER}/${nip}~${docker}.*
rm -f ${TMPFOLDER}/${nip}-${docker}.errtime
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE}
else else
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}-${docker}.down ]; then if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1 SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1
touch ${TMPFOLDER}/${nip}-${docker}.down touch ${TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${docker}.down) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${TMPFOLDER}/${nip}-${docker}.errtime ]; then if [ ! -f ${TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${TMPFOLDER}/${nip}-${docker}.down ${TMPFOLDER}/${nip}-${docker}.errtime mv ${TMPFOLDER}/${nip}~${docker}.down ${TMPFOLDER}/${nip}~${docker}.errtime
fi fi
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${docker}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.errtime)
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi fi
touch ${TMPFOLDER}/${nip}-${docker}.down touch ${TMPFOLDER}/${nip}~${docker}.down
fi fi
fi fi
@@ -383,8 +382,7 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}" echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi fi
rm -f ${TMPFOLDER}/${nip}-${docker}.down rm -f ${TMPFOLDER}/${nip}~${docker}.*
rm -f ${TMPFOLDER}/${nip}-${docker}.errtime
fi fi
fi fi
done done