From f549a446c35ad383379bdffedb9b7fef26c39184 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Fri, 8 Feb 2019 12:27:43 -0600 Subject: [PATCH] Update nodemgmt-scripts.sh --- nodemgmt-scripts.sh | 322 ++++++++++++++++++++++---------------------- 1 file changed, 163 insertions(+), 159 deletions(-) diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index c3f965c2..65214ee9 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -292,125 +292,125 @@ STATUS(){ echo "runtime: ${runtime}" } -STATUS-CHECK(){ - start=`date +%s` - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then - if [ ! -z ${LOCAL_SERVICES+x} ]; then NHCMD='ssh root@10.5.10.51' - else NHCMD=''; fi - # for rcheck in "${REPL_CHECK[@]}"; do - # ${NHCMD} rm -f ${FOLDER}/test.repl - # daterun=`date +%Y-%m-%d-%H-%M-%S` - # if [ "${NHCMD}" = "" ]; then - # echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl - # yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl - # else - # ${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl" - # yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl" - # fi - # done - if [ ! -z ${LOCAL_SERVICES+x} ]; then - lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) - for srvc in "${LOCAL_SERVICES[@]}"; do - if [ "$(systemctl is-active ${srvc})" != "active" ]; then - if [ ! -f ${FOLDER}/localhost-${srvc}.down ]; then - echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} - touch ${FOLDER}/localhost-${srvc}.down - fi - elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then - echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} - rm -f ${FOLDER}/localhost-${srvc}.down - rm -f ${FOLDER}/localhost-${srvc}.errtime - fi - done - fi - if [ -z ${LOCAL_SERVICES+x} ] || [ "${1}" = "all" ]; then - nid=1 - for nip in "${NODE_HOSTS[@]}"; do - checkhost=$(CHECK_HOST ${nip}) - if [ "${checkhost}" != "false" ]; then - if [ -f ${FOLDER}/${nip}.down ]; then - rm -f ${FOLDER}/${nip}.down - echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL} - fi - if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' - else NCMD="ssh root@${nip}" - fi - for srvc in "${NODE_SERVICES[@]}"; do - if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then - if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then - echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} - touch ${FOLDER}/${nip}-${srvc}.down - else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime - fi - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} - touch ${FOLDER}/${nip}-${srvc}.down - fi - fi - elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then - echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} - rm -f ${FOLDER}/${nip}-${srvc}.down - rm -f ${FOLDER}/${nip}-${srvc}.errtime - fi - done - elif [ ! -f ${FOLDER}/${nip}.down ]; then - touch ${FOLDER}/${nip}.down - echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL} - fi - nid=`expr $nid + 1` - done - ##REPLICATION CHECK - # nid=1 - # for nip in "${NODE_HOSTS[@]}"; do - # if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi - # if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then - # checkhost=$(CHECK_HOST ${nip}) - # if [ "${checkhost}" != "false" ]; then - # if [ -f ${FOLDER}/${nip}.down ]; then - # rm -f ${FOLDER}/${nip}.down - # echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL} - # fi - # for rcheck in "${REPL_CHECK[@]}"; do - # timeout=`date --date='20 seconds' +%s` - # checked=false - # until [ "${checked}" = "" ]; do - # if [ "${NHCMD}" = "" ]; then - # checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` - # else - # checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" - # fi - # if [ "`date +%s`" -gt "$timeout" ]; then - # timeout=true - # break - # fi - # done - # if [ "${timeout}" != "true" ]; then - # if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then - # rm -f ${FOLDER}/${nip}-${rcheck}.down - # echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL} - # fi - # elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then - # touch ${FOLDER}/${nip}-${rcheck}.down - # echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL} - # fi - # done - # elif [ ! -f ${FOLDER}/${nip}.down ]; then - # touch ${FOLDER}/${nip}.down - # echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL} - # fi - # fi - # nid=`expr $nid + 1` - # done - fi - fi - end=`date +%s` - runtime=$((end-start)) - echo "runtime: ${runtime}" -} +# STATUS-CHECK(){ +# start=`date +%s` +# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then +# if [ ! -z ${LOCAL_SERVICES+x} ]; then NHCMD='ssh root@10.5.10.51' +# else NHCMD=''; fi +# # for rcheck in "${REPL_CHECK[@]}"; do +# # ${NHCMD} rm -f ${FOLDER}/test.repl +# # daterun=`date +%Y-%m-%d-%H-%M-%S` +# # if [ "${NHCMD}" = "" ]; then +# # echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl +# # yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl +# # else +# # ${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl" +# # yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl" +# # fi +# # done +# if [ ! -z ${LOCAL_SERVICES+x} ]; then +# lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) +# for srvc in "${LOCAL_SERVICES[@]}"; do +# if [ "$(systemctl is-active ${srvc})" != "active" ]; then +# if [ ! -f ${FOLDER}/localhost-${srvc}.down ]; then +# echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} +# touch ${FOLDER}/localhost-${srvc}.down +# fi +# elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then +# echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} +# rm -f ${FOLDER}/localhost-${srvc}.down +# rm -f ${FOLDER}/localhost-${srvc}.errtime +# fi +# done +# fi +# if [ -z ${LOCAL_SERVICES+x} ] || [ "${1}" = "all" ]; then +# nid=1 +# for nip in "${NODE_HOSTS[@]}"; do +# checkhost=$(CHECK_HOST ${nip}) +# if [ "${checkhost}" != "false" ]; then +# if [ -f ${FOLDER}/${nip}.down ]; then +# rm -f ${FOLDER}/${nip}.down +# echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL} +# fi +# if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' +# else NCMD="ssh root@${nip}" +# fi +# for srvc in "${NODE_SERVICES[@]}"; do +# if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then +# if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then +# echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} +# touch ${FOLDER}/${nip}-${srvc}.down +# else +# errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) +# if [ $errtime -gt ${RENOTIFY} ]; then +# if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then +# mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime +# fi +# toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) +# echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} +# touch ${FOLDER}/${nip}-${srvc}.down +# fi +# fi +# elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then +# echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} +# rm -f ${FOLDER}/${nip}-${srvc}.down +# rm -f ${FOLDER}/${nip}-${srvc}.errtime +# fi +# done +# elif [ ! -f ${FOLDER}/${nip}.down ]; then +# touch ${FOLDER}/${nip}.down +# echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL} +# fi +# nid=`expr $nid + 1` +# done +# ##REPLICATION CHECK +# # nid=1 +# # for nip in "${NODE_HOSTS[@]}"; do +# # if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi +# # if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then +# # checkhost=$(CHECK_HOST ${nip}) +# # if [ "${checkhost}" != "false" ]; then +# # if [ -f ${FOLDER}/${nip}.down ]; then +# # rm -f ${FOLDER}/${nip}.down +# # echo "iDS-Node${nid} is back UP!" | mail -s "iDS-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL} +# # fi +# # for rcheck in "${REPL_CHECK[@]}"; do +# # timeout=`date --date='20 seconds' +%s` +# # checked=false +# # until [ "${checked}" = "" ]; do +# # if [ "${NHCMD}" = "" ]; then +# # checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` +# # else +# # checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" +# # fi +# # if [ "`date +%s`" -gt "$timeout" ]; then +# # timeout=true +# # break +# # fi +# # done +# # if [ "${timeout}" != "true" ]; then +# # if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then +# # rm -f ${FOLDER}/${nip}-${rcheck}.down +# # echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL} +# # fi +# # elif [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then +# # touch ${FOLDER}/${nip}-${rcheck}.down +# # echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'iDS-Node${nid}'" ${STATUS_CHECK_EMAIL} +# # fi +# # done +# # elif [ ! -f ${FOLDER}/${nip}.down ]; then +# # touch ${FOLDER}/${nip}.down +# # echo "iDS-Node${nid} is down" | mail -s "iDS-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL} +# # fi +# # fi +# # nid=`expr $nid + 1` +# # done +# fi +# fi +# end=`date +%s` +# runtime=$((end-start)) +# echo "runtime: ${runtime}" +# } NEWCERT(){ echo "" @@ -430,41 +430,45 @@ NEWCERT(){ if [ "${NEW_CERT}" != "" ]; then echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}" echo "" - do_with_root $CERT_DAEMON certonly --webroot -w /var/www/html -d ${NEW_CERT} + $CERT_DAEMON certonly --webroot -w /var/www/html -d ${NEW_CERT} if [ -f /etc/letsencrypt/live/${MAIN_CERT}/cert.pem ]; then - do_with_root chown -R root:letsencrypt /etc/letsencrypt - do_with_root chmod -R 6775 /etc/letsencrypt - # rm -f ${FOLDER}/cert-request.lastrun - # daterun=`date +%Y-%m-%d-%H-%M-%S` - # echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun - # yes | cp -rfH ${FOLDER}/cert-request.lastrun /etc/letsencrypt/cert-request.lastrun - # DIVIDER true - # echo -en "${idsCL[LightYellow]}Waiting for certificate replication between the nodes... ${idsCL[Default]}" - # checked=false - # timeout=`date --date='2 minutes' +%s` - # until [ "${checked}" = "" ]; do - # tchecked='' - # for nip in "${NODE_HOSTS[@]}"; do - # if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' - # else NCMD="ssh root@${nip}" - # fi - # if [ "${NCMD}" != "" ]; then - # tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-request.lastrun" | diff - /etc/letsencrypt/cert-request.lastrun` - # fi - # done - # checked=${tchecked} - # if [ "`date +%s`" -gt "$timeout" ]; then - # echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" - # timeout=true - # echo "Timeout occurred in waiting for replication between nodes. (${NEW_CERT})" | mail -s "Cert-Request" ${STATUS_CHECK_EMAIL} - # break - # fi - # done - # if [ "${timeout}" != "true" ]; then - # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" - # fi - # echo - # SERVICE nginx reload + chown -R root:letsencrypt /etc/letsencrypt + chmod -R 6775 /etc/letsencrypt + if [ -f /etc/nginx/sites-available/${MAIN_CERT} ]; then + rm -f ${FOLDER}/cert-request.lastrun + daterun=`date +%Y-%m-%d-%H-%M-%S` + echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun + yes | cp -rfH ${FOLDER}/cert-request.lastrun /etc/letsencrypt/cert-request.lastrun + DIVIDER true + echo -e "Since this certificate is tied to a site in NGINX, we will reload NGINX across the nodes." + echo + echo -en "${idsCL[LightYellow]}Waiting for certificate replication across the nodes... ${idsCL[Default]}" + checked=false + timeout=`date --date='2 minutes' +%s` + until [ "${checked}" = "" ]; do + tchecked='' + for nip in "${NODE_HOSTS[@]}"; do + if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' + else NCMD="ssh root@${nip}" + fi + if [ "${NCMD}" != "" ]; then + tchecked+=`${NCMD} "cat /etc/letsencrypt/cert-request.lastrun" | diff - /etc/letsencrypt/cert-request.lastrun` + fi + done + checked=${tchecked} + if [ "`date +%s`" -gt "$timeout" ]; then + echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" + timeout=true + echo "Timeout occurred in waiting for replication between nodes. (${NEW_CERT})" | mail -s "Cert-Request" ${STATUS_CHECK_EMAIL} + break + fi + done + if [ "${timeout}" != "true" ]; then + echo -e "${idsCL[Green]}Completed${idsCL[Default]}" + fi + echo + SERVICE nginx reload + fi echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}" else echo "" @@ -486,9 +490,9 @@ NEWCERT(){ CERTRENEW(){ echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}" echo - do_with_root $CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun - do_with_root chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun - do_with_root chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun + $CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun + chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun + chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun @@ -530,9 +534,9 @@ CERTRENEW(){ } NIGHTLYRENEW(){ rm -f ${FOLDER}/cert-renewal.lastrun - do_with_root $CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun - do_with_root chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun - do_with_root chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun + $CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun + chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun + chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun