From 9db77138306ccf6f44912b348ae1144dfab4ad28 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Sun, 19 Mar 2023 10:34:25 -0500 Subject: [PATCH] Update nodemgmt-scripts.sh --- nodemgmt-scripts.sh | 53 ++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 06a44209..89a5b947 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -93,7 +93,8 @@ STATUS(){ if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\n$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi + #if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\n$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi + SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1 touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} else @@ -120,7 +121,8 @@ STATUS(){ else toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down) fi - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi + #if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi + SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE} @@ -183,7 +185,8 @@ STATUS(){ toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) fi rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime - if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${EMAIL_NOTICE}; fi + #if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${EMAIL_NOTICE}; fi + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} fi @@ -226,7 +229,8 @@ STATUS(){ if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi + # if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 touch ${FOLDER}/${nip}-${srvc}.down echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} @@ -257,7 +261,8 @@ STATUS(){ else toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) fi - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi + # if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then @@ -303,7 +308,8 @@ STATUS(){ if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" fi - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi + # if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed" rm -f ${FOLDER}/${nip}-${docker}.down rm -f ${FOLDER}/${nip}-${docker}.errtime echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE} @@ -313,7 +319,8 @@ STATUS(){ echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi + # if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1 touch ${FOLDER}/${nip}-${docker}.down echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE} else @@ -352,7 +359,8 @@ STATUS(){ toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node is down!${idsCL[Default]}" fi - [ "${EMAIL_NOTICE}" != "" ] && echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE} + # [ "${EMAIL_NOTICE}" != "" ] && echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE} + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1 echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE} else if [ "${ST_ACTION}" != "check" ]; then @@ -466,7 +474,8 @@ STATUS(){ fi if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then touch ${FOLDER}/${nip}-${rcheck}.down - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\n$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi + # if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\n$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi + SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1 echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} fi else @@ -475,7 +484,8 @@ STATUS(){ fi if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then rm -f ${FOLDER}/${nip}-${rcheck}.down - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\n$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi + # if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\n$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi + SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1 echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} fi fi @@ -564,13 +574,15 @@ STATUS(){ if [ "${freespace}" -le "1000" ]; then fs_status='error' fs_status_color='Red' - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\n$(date)" | mail -s "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi + # if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\n$(date)" | mail -s "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi + SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" 1 elif [ "${freespace}" -le "5000" ]; then fs_status='warn' fs_status_color='Yellow' - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\n$(date)" | mail -s "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi - + # if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\n$(date)" | mail -s "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi + SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" + else fs_status='' fs_status_color='Green' @@ -1071,7 +1083,8 @@ CHECK-CERTS(){ ${FOLDER}/ssl-cert-check/ssl-cert-check -a -f ${FOLDER}/ssl-cert-check/ssldomains > ${FOLDER}/ssl-cert-check.lastrun if [ "${EMAIL_NOTICE}" != "" ]; then SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)" - [ "${EMAIL_NOTICE}" != "" ] && echo -e "SSL Certs Check\n$SSLLOG" | mail -s "SSL Certs Check" ${EMAIL_NOTICE} + # [ "${EMAIL_NOTICE}" != "" ] && echo -e "SSL Certs Check\n$SSLLOG" | mail -s "SSL Certs Check" ${EMAIL_NOTICE} + SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG" fi else @@ -2081,7 +2094,8 @@ SINGLE_SERVER_SERVICE_CHECK(){ if [ -f /mnt/web-data/mounted ]; then for stc in "${services_to_check[@]}"; do if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then - if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes\n$(date)" | mail -s "Node-Balancing Notice" ${EMAIL_NOTICE}; fi + # if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes\n$(date)" | mail -s "Node-Balancing Notice" ${EMAIL_NOTICE}; fi + SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes" echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${LOGFILE} [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service" @@ -2334,7 +2348,8 @@ UPDATE_DYNDNS(){ /usr/bin/curl -s "https://systems:Dcs9613@pdns.scity.us/nic/update?hostname=snsinvest.com&myip=${NEWIP}" > /dev/null 2>&1 /usr/bin/curl -s "https://systems:Dcs9613@pdns.scity.us/nic/update?hostname=misfitsmc.com&myip=${NEWIP}" > /dev/null 2>&1 - [ "${EMAIL_NOTICE}" != "" ] && echo "DYNDNS Updated" | mail -s "Updated WAN IP from '${OLDIP}' to '${NEWIP}'" ${EMAIL_NOTICE} + # [ "${EMAIL_NOTICE}" != "" ] && echo "DYNDNS Updated" | mail -s "Updated WAN IP from '${OLDIP}' to '${NEWIP}'" ${EMAIL_NOTICE} + SENDNOTICE "Updated WAN IP from '${OLDIP}' to '${NEWIP}'" "DYNDNS Updated" if [ "${1}" != 'q' ]; then echo -e "${idsCL[Green]}Updated WAN IP from '${OLDIP}' to '${NEWIP}'${idsCL[Default]}" @@ -2516,7 +2531,8 @@ if [ ${action-x} ]; then rm -f ${FOLDER}/status-downlist.lastrun DOWNLIST report >> ${FOLDER}/status-downlist.lastrun STATUSLOG="$(cat ${FOLDER}/status-downlist.lastrun)" - [ "${EMAIL_NOTICE}" != "" ] && echo -e "Downlist Report\n${STATUSLOG}" | mail -s "Downlist Report" ${EMAIL_NOTICE} + # [ "${EMAIL_NOTICE}" != "" ] && echo -e "Downlist Report\n${STATUSLOG}" | mail -s "Downlist Report" ${EMAIL_NOTICE} + SENDNOTICE "Downlist Report" "Downlist Report\n${STATUSLOG}" fi else DOWNLIST ${2} @@ -2536,7 +2552,8 @@ if [ ${action-x} ]; then echo -e "\n\n" >> ${FOLDER}/status-check.lastrun DOWNLIST report >> ${FOLDER}/status-check.lastrun STATUSLOG="$(cat ${FOLDER}/status-check.lastrun)" - [ "${EMAIL_NOTICE}" != "" ] && echo -e "Status Report\n${STATUSLOG}" | mail -s "Status Report" ${EMAIL_NOTICE} + # [ "${EMAIL_NOTICE}" != "" ] && echo -e "Status Report\n${STATUSLOG}" | mail -s "Status Report" ${EMAIL_NOTICE} + SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}" fi else STATUS ${2} ${3}