From 4f4b5098931de09f064b7c976f856de5b74d6857 Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Fri, 3 Mar 2023 22:23:54 -0600
Subject: [PATCH] Update nodemgmt-scripts.sh

---
 nodemgmt-scripts.sh | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh
index fb07d579..06a44209 100755
--- a/nodemgmt-scripts.sh
+++ b/nodemgmt-scripts.sh
@@ -93,7 +93,7 @@ STATUS(){
 						if [ "${ST_ACTION}" != "check" ]; then
 							echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
 						fi
-						if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi
+						if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\n$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi
 						touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
 						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
 					else
@@ -120,7 +120,7 @@ STATUS(){
 						else
 							toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
 						fi
-						if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi
+						if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NODE_HOSTNAME}" ${EMAIL_NOTICE}; fi
 						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
 						rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
 						echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE}
@@ -183,7 +183,7 @@ STATUS(){
 								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
 							fi
 							rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
-							if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${EMAIL_NOTICE}; fi
+							if [ "${EMAIL_NOTICE}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${EMAIL_NOTICE}; fi
 							echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
 
 						fi
@@ -226,7 +226,7 @@ STATUS(){
 												if [ "${ST_ACTION}" != "check" ]; then
 													echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
 												fi
-												if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
+												if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
 												touch ${FOLDER}/${nip}-${srvc}.down
 												echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
 												
@@ -257,7 +257,7 @@ STATUS(){
 											else
 												toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
 											fi
-											if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
+											if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
 											echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
 
 											if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
@@ -303,7 +303,7 @@ STATUS(){
 										if [ "${ST_ACTION}" != "check" ]; then
 											echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
 										fi
-										if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
+										if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
 										rm -f ${FOLDER}/${nip}-${docker}.down
 										rm -f ${FOLDER}/${nip}-${docker}.errtime
 										echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE}
@@ -313,7 +313,7 @@ STATUS(){
 											echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
 										
 											if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
-												if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
+												if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}; fi
 												touch ${FOLDER}/${nip}-${docker}.down
 												echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE}
 											else
@@ -352,7 +352,7 @@ STATUS(){
 								toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
 								echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node is down!${idsCL[Default]}"
 							fi
-							[ "${EMAIL_NOTICE}" != "" ] && echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}
+							[ "${EMAIL_NOTICE}" != "" ] && echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down\n$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}
 							echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE}
 						else
 							if [ "${ST_ACTION}" != "check" ]; then
@@ -466,7 +466,7 @@ STATUS(){
 										fi
 										if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
 											touch ${FOLDER}/${nip}-${rcheck}.down
-											if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\t$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
+											if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\n$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
 											echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
 										fi
 									else
@@ -475,7 +475,7 @@ STATUS(){
 										fi
 										if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
 											rm -f ${FOLDER}/${nip}-${rcheck}.down
-											if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\t$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
+											if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})\n$(date)" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
 											echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
 										fi
 									fi
@@ -564,12 +564,12 @@ STATUS(){
 							if [ "${freespace}" -le "1000" ]; then
 								fs_status='error'
 								fs_status_color='Red'
-								if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\t$(date)" | mail -s "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
+								if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\n$(date)" | mail -s "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
 
 							elif [ "${freespace}" -le "5000" ]; then
 								fs_status='warn'
 								fs_status_color='Yellow'
-								if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\t$(date)" | mail -s "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
+								if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "${partition} : ${freespace} MB free\n$(date)" | mail -s "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${EMAIL_NOTICE}; fi
 								
 							else
 								fs_status=''
@@ -2081,7 +2081,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
 			if [ -f /mnt/web-data/mounted ]; then
 				for stc in "${services_to_check[@]}"; do
 					if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
-						if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes" | mail -s "Node-Balancing Notice" ${EMAIL_NOTICE}; fi
+						if [ "${EMAIL_NOTICE}" != "" ]; then echo -e "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes\n$(date)" | mail -s "Node-Balancing Notice" ${EMAIL_NOTICE}; fi
 						echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${LOGFILE}
 						
 						[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"