Update nodemgmt-scripts.sh

This commit is contained in:
2019-02-08 12:27:43 -06:00
parent 82f5a7821b
commit f549a446c3

View File

@@ -292,125 +292,125 @@ STATUS(){
echo "runtime: ${runtime}" echo "runtime: ${runtime}"
} }
STATUS-CHECK(){ # STATUS-CHECK(){
start=`date +%s` # start=`date +%s`
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
if [ ! -z ${LOCAL_SERVICES+x} ]; then NHCMD='ssh root@10.5.10.51' # if [ ! -z ${LOCAL_SERVICES+x} ]; then NHCMD='ssh root@10.5.10.51'
else NHCMD=''; fi # else NHCMD=''; fi
# for rcheck in "${REPL_CHECK[@]}"; do # # for rcheck in "${REPL_CHECK[@]}"; do
# ${NHCMD} rm -f ${FOLDER}/test.repl # # ${NHCMD} rm -f ${FOLDER}/test.repl
# daterun=`date +%Y-%m-%d-%H-%M-%S` # # daterun=`date +%Y-%m-%d-%H-%M-%S`
# if [ "${NHCMD}" = "" ]; then # # if [ "${NHCMD}" = "" ]; then
# echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl # # echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
# yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl # # yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
# else # # else
# ${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl" # # ${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
# yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl" # # yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
# # fi
# # done
# if [ ! -z ${LOCAL_SERVICES+x} ]; then
# lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
# for srvc in "${LOCAL_SERVICES[@]}"; do
# if [ "$(systemctl is-active ${srvc})" != "active" ]; then
# if [ ! -f ${FOLDER}/localhost-${srvc}.down ]; then
# echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
# touch ${FOLDER}/localhost-${srvc}.down
# fi
# elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then
# echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
# rm -f ${FOLDER}/localhost-${srvc}.down
# rm -f ${FOLDER}/localhost-${srvc}.errtime
# fi # fi
# done # done
if [ ! -z ${LOCAL_SERVICES+x} ]; then # fi
lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) # if [ -z ${LOCAL_SERVICES+x} ] || [ "${1}" = "all" ]; then
for srvc in "${LOCAL_SERVICES[@]}"; do
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/localhost-${srvc}.down ]; then
echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
touch ${FOLDER}/localhost-${srvc}.down
fi
elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then
echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL}
rm -f ${FOLDER}/localhost-${srvc}.down
rm -f ${FOLDER}/localhost-${srvc}.errtime
fi
done
fi
if [ -z ${LOCAL_SERVICES+x} ] || [ "${1}" = "all" ]; then
nid=1
for nip in "${NODE_HOSTS[@]}"; do
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
rm -f ${FOLDER}/${nip}.down
echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
fi
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
else NCMD="ssh root@${nip}"
fi
for srvc in "${NODE_SERVICES[@]}"; do
if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
touch ${FOLDER}/${nip}-${srvc}.down
fi
fi
elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
done
elif [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
fi
nid=`expr $nid + 1`
done
##REPLICATION CHECK
# nid=1 # nid=1
# for nip in "${NODE_HOSTS[@]}"; do # for nip in "${NODE_HOSTS[@]}"; do
# if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
# if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
# checkhost=$(CHECK_HOST ${nip}) # checkhost=$(CHECK_HOST ${nip})
# if [ "${checkhost}" != "false" ]; then # if [ "${checkhost}" != "false" ]; then
# if [ -f ${FOLDER}/${nip}.down ]; then # if [ -f ${FOLDER}/${nip}.down ]; then
# rm -f ${FOLDER}/${nip}.down # rm -f ${FOLDER}/${nip}.down
# echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL} # echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
# fi # fi
# for rcheck in "${REPL_CHECK[@]}"; do # if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
# timeout=`date --date='20 seconds' +%s` # else NCMD="ssh root@${nip}"
# checked=false # fi
# until [ "${checked}" = "" ]; do # for srvc in "${NODE_SERVICES[@]}"; do
# if [ "${NHCMD}" = "" ]; then # if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then
# checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` # if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
# echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# touch ${FOLDER}/${nip}-${srvc}.down
# else # else
# checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" # errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
# if [ $errtime -gt ${RENOTIFY} ]; then
# if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
# mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
# fi # fi
# if [ "`date +%s`" -gt "$timeout" ]; then # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
# timeout=true # echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# break # touch ${FOLDER}/${nip}-${srvc}.down
# fi # fi
# done
# if [ "${timeout}" != "true" ]; then
# if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
# rm -f ${FOLDER}/${nip}-${rcheck}.down
# echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
# fi # fi
# elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then # elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
# touch ${FOLDER}/${nip}-${rcheck}.down # echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
# echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL} # rm -f ${FOLDER}/${nip}-${srvc}.down
# rm -f ${FOLDER}/${nip}-${srvc}.errtime
# fi # fi
# done # done
# elif [ ! -f ${FOLDER}/${nip}.down ]; then # elif [ ! -f ${FOLDER}/${nip}.down ]; then
# touch ${FOLDER}/${nip}.down # touch ${FOLDER}/${nip}.down
# echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL} # echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
# fi # fi
# fi
# nid=`expr $nid + 1` # nid=`expr $nid + 1`
# done # done
fi # ##REPLICATION CHECK
fi # # nid=1
end=`date +%s` # # for nip in "${NODE_HOSTS[@]}"; do
runtime=$((end-start)) # # if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
echo "runtime: ${runtime}" # # if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
} # # checkhost=$(CHECK_HOST ${nip})
# # if [ "${checkhost}" != "false" ]; then
# # if [ -f ${FOLDER}/${nip}.down ]; then
# # rm -f ${FOLDER}/${nip}.down
# # echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}
# # fi
# # for rcheck in "${REPL_CHECK[@]}"; do
# # timeout=`date --date='20 seconds' +%s`
# # checked=false
# # until [ "${checked}" = "" ]; do
# # if [ "${NHCMD}" = "" ]; then
# # checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
# # else
# # checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
# # fi
# # if [ "`date +%s`" -gt "$timeout" ]; then
# # timeout=true
# # break
# # fi
# # done
# # if [ "${timeout}" != "true" ]; then
# # if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
# # rm -f ${FOLDER}/${nip}-${rcheck}.down
# # echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
# # fi
# # elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
# # touch ${FOLDER}/${nip}-${rcheck}.down
# # echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL}
# # fi
# # done
# # elif [ ! -f ${FOLDER}/${nip}.down ]; then
# # touch ${FOLDER}/${nip}.down
# # echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}
# # fi
# # fi
# # nid=`expr $nid + 1`
# # done
# fi
# fi
# end=`date +%s`
# runtime=$((end-start))
# echo "runtime: ${runtime}"
# }
NEWCERT(){ NEWCERT(){
echo "" echo ""
@@ -430,41 +430,45 @@ NEWCERT(){
if [ "${NEW_CERT}" != "" ]; then if [ "${NEW_CERT}" != "" ]; then
echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}" echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}"
echo "" echo ""
do_with_root $CERT_DAEMON certonly --webroot -w /var/www/html -d ${NEW_CERT} $CERT_DAEMON certonly --webroot -w /var/www/html -d ${NEW_CERT}
if [ -f /etc/letsencrypt/live/${MAIN_CERT}/cert.pem ]; then if [ -f /etc/letsencrypt/live/${MAIN_CERT}/cert.pem ]; then
do_with_root chown -R root:letsencrypt /etc/letsencrypt chown -R root:letsencrypt /etc/letsencrypt
do_with_root chmod -R 6775 /etc/letsencrypt chmod -R 6775 /etc/letsencrypt
# rm -f ${FOLDER}/cert-request.lastrun if [ -f /etc/nginx/sites-available/${MAIN_CERT} ]; then
# daterun=`date +%Y-%m-%d-%H-%M-%S` rm -f ${FOLDER}/cert-request.lastrun
# echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S`
# yes | cp -rfH ${FOLDER}/cert-request.lastrun /etc/letsencrypt/cert-request.lastrun echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun
# DIVIDER true yes | cp -rfH ${FOLDER}/cert-request.lastrun /etc/letsencrypt/cert-request.lastrun
# echo -en "${idsCL[LightYellow]}Waiting for certificate replication between the nodes... ${idsCL[Default]}" DIVIDER true
# checked=false echo -e "Since this certificate is tied to a site in NGINX, we will reload NGINX across the nodes."
# timeout=`date --date='2 minutes' +%s` echo
# until [ "${checked}" = "" ]; do echo -en "${idsCL[LightYellow]}Waiting for certificate replication across the nodes... ${idsCL[Default]}"
# tchecked='' checked=false
# for nip in "${NODE_HOSTS[@]}"; do timeout=`date --date='2 minutes' +%s`
# if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' until [ "${checked}" = "" ]; do
# else NCMD="ssh root@${nip}" tchecked=''
# fi for nip in "${NODE_HOSTS[@]}"; do
# if [ "${NCMD}" != "" ]; then if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''
# tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-request.lastrun" | diff - /etc/letsencrypt/cert-request.lastrun` else NCMD="ssh root@${nip}"
# fi fi
# done if [ "${NCMD}" != "" ]; then
# checked=${tchecked} tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-request.lastrun" | diff - /etc/letsencrypt/cert-request.lastrun`
# if [ "`date +%s`" -gt "$timeout" ]; then fi
# echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" done
# timeout=true checked=${tchecked}
# echo "Timeout occurred in waiting for replication between nodes. (${NEW_CERT})" | mail -s "Cert-Request" ${STATUS_CHECK_EMAIL} if [ "`date +%s`" -gt "$timeout" ]; then
# break echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
# fi timeout=true
# done echo "Timeout occurred in waiting for replication between nodes. (${NEW_CERT})" | mail -s "Cert-Request" ${STATUS_CHECK_EMAIL}
# if [ "${timeout}" != "true" ]; then break
# echo -e "${idsCL[Green]}Completed${idsCL[Default]}" fi
# fi done
# echo if [ "${timeout}" != "true" ]; then
# SERVICE nginx reload echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
fi
echo
SERVICE nginx reload
fi
echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}" echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}"
else else
echo "" echo ""
@@ -486,9 +490,9 @@ NEWCERT(){
CERTRENEW(){ CERTRENEW(){
echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}" echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}"
echo echo
do_with_root $CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun $CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
do_with_root chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
do_with_root chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S` daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
@@ -530,9 +534,9 @@ CERTRENEW(){
} }
NIGHTLYRENEW(){ NIGHTLYRENEW(){
rm -f ${FOLDER}/cert-renewal.lastrun rm -f ${FOLDER}/cert-renewal.lastrun
do_with_root $CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun $CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun
do_with_root chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
do_with_root chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S` daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun