voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update nodemgmt-scripts.sh

Browse Source
This commit is contained in:
David Schroeder
2023-03-19 11:31:19 -05:00
parent 6d4a6a2d51
commit 967faa0fd7
1 changed files with 0 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
nodemgmt-scripts.sh
View File

@@ -93,7 +93,6 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
#if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\n$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1
touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
@@ -107,7 +106,6 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
# if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi
touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
fi
fi
@@ -121,7 +119,6 @@ STATUS(){
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
fi
#if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
@@ -227,7 +224,6 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
# if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
touch ${FOLDER}/${nip}-${srvc}.down
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
@@ -239,7 +235,6 @@ STATUS(){
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
# if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
touch ${FOLDER}/${nip}-${srvc}.down
if [ "${srvc}" = "mysql" ]; then
${NCMD} systemctl restart ${srvc}
@@ -259,7 +254,6 @@ STATUS(){
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
# if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
@@ -306,7 +300,6 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
# if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed"
rm -f ${FOLDER}/${nip}-${docker}.down
rm -f ${FOLDER}/${nip}-${docker}.errtime
@@ -317,7 +310,6 @@ STATUS(){
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
# if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1
touch ${FOLDER}/${nip}-${docker}.down
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE}
@@ -357,7 +349,6 @@ STATUS(){
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node is down!${idsCL[Default]}"
fi
# [ "${EMAIL_NOTICE}" != "" ] && echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE}
else
@@ -479,7 +470,6 @@ STATUS(){
fi
if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down
# if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\n$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi
@@ -489,7 +479,6 @@ STATUS(){
fi
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
rm -f ${FOLDER}/${nip}-${rcheck}.down
# if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\n$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi
@@ -502,7 +491,6 @@ STATUS(){
fi
if [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
# if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-DOWN" ${EMAIL_NOTICE}; fi
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
@@ -579,13 +567,11 @@ STATUS(){
if [ "${freespace}" -le "1000" ]; then
fs_status='error'
fs_status_color='Red'
# if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\n$(date)" | mail -s "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" 1
elif [ "${freespace}" -le "5000" ]; then
fs_status='warn'
fs_status_color='Yellow'
# if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\n$(date)" | mail -s "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free"
else
@@ -1090,7 +1076,6 @@ CHECK-CERTS(){
${FOLDER}/ssl-cert-check/ssl-cert-check -a -f ${FOLDER}/ssl-cert-check/ssldomains > ${FOLDER}/ssl-cert-check.lastrun
if [ "${EMAIL_NOTICE}" != "" ]; then
SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)"
# [ "${EMAIL_NOTICE}" != "" ] && echo -e "SSL Certs Check\n$SSLLOG" | mail -s "SSL Certs Check" ${EMAIL_NOTICE}
SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG"
fi
@@ -2101,7 +2086,6 @@ SINGLE_SERVER_SERVICE_CHECK(){
if [ -f /mnt/web-data/mounted ]; then
for stc in "${services_to_check[@]}"; do
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
# if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes\n$(date)" | mail -s "Node-Balancing Notice" ${EMAIL_NOTICE}; fi
SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${LOGFILE}
@@ -2355,7 +2339,6 @@ UPDATE_DYNDNS(){
/usr/bin/curl -s "https://systems:Dcs9613@pdns.scity.us/nic/update?hostname=snsinvest.com&myip=${NEWIP}" > /dev/null 2>&1
/usr/bin/curl -s "https://systems:Dcs9613@pdns.scity.us/nic/update?hostname=misfitsmc.com&myip=${NEWIP}" > /dev/null 2>&1
# [ "${EMAIL_NOTICE}" != "" ] && echo "DYNDNS Updated" | mail -s "Updated WAN IP from '${OLDIP}' to '${NEWIP}'" ${EMAIL_NOTICE}
SENDNOTICE "Updated WAN IP from '${OLDIP}' to '${NEWIP}'" "DYNDNS Updated"
if [ "${1}" != 'q' ]; then
@@ -2538,7 +2521,6 @@ if [ ${action-x} ]; then
rm -f ${FOLDER}/status-downlist.lastrun
DOWNLIST report >> ${FOLDER}/status-downlist.lastrun
STATUSLOG="$(cat ${FOLDER}/status-downlist.lastrun)"
# [ "${EMAIL_NOTICE}" != "" ] && echo -e "Downlist Report\n${STATUSLOG}" | mail -s "Downlist Report" ${EMAIL_NOTICE}
SENDNOTICE "Downlist Report" "Downlist Report\n${STATUSLOG}"
fi
else
@@ -2559,7 +2541,6 @@ if [ ${action-x} ]; then
echo -e "\n\n" >> ${FOLDER}/status-check.lastrun
DOWNLIST report >> ${FOLDER}/status-check.lastrun
STATUSLOG="$(cat ${FOLDER}/status-check.lastrun)"
# [ "${EMAIL_NOTICE}" != "" ] && echo -e "Status Report\n${STATUSLOG}" | mail -s "Status Report" ${EMAIL_NOTICE}
SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}"
fi
else