diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 5c613cd7..9e84d740 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -61,6 +61,45 @@ STATUS() { ENTER2CONTINUE fi } + +STATUS-CHECK(){ + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then + if [ ! -z ${LOCAL_SERVICES+x} ]; then + lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) + for srvc in "${LOCAL_SERVICES[@]}" + do + if [ $(pgrep ${srvc} | wc -l) -lt "1" ]; then + echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} + touch ${FOLDER}/localhost-${srvc}.down + elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then + echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} + rm -f ${FOLDER}/localhost-${srvc}.down + fi + done + fi + if [ -z ${LOCAL_SERVICES+x} ] || [ "${2}" = "all" ]; then + nid=1 + for nip in "${NODE_HOSTS[@]}" + do + if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' + else NCMD="ssh root@${nip}" + fi + for srvc in "${NODE_SERVICES[@]}" + do + if [ $(${NCMD} pgrep ${srvc} | wc -l) -lt "1" ]; then + echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} + touch ${FOLDER}/${nip}-${srvc}.down + elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then + echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} + rm -f ${FOLDER}/${nip}-${srvc}.down + fi + done + nid=`expr $nid + 1` + done + fi + fi +} + NEWCERT(){ echo "" if [ -z ${1+x} ]; then @@ -171,9 +210,8 @@ NIGHTLYRENEW(){ done checked=${tchecked} if [ "`date +%s`" -gt "$timeout" ]; then - echo 'timeout' - - exit 1 + echo "Timeout occured in waiting for replication between nodes." | mail -s "Cert-Renewal" ${STATUS_CHECK_EMAIL} + break fi done SERVICE nginx reload &>> /opt/idssys/nodemgmt/cert-renewal.lastrun @@ -593,44 +631,6 @@ NEWSITE(){ fi } -STATUS-CHECK(){ - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then - if [ ! -z ${LOCAL_SERVICES+x} ]; then - lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) - for srvc in "${LOCAL_SERVICES[@]}" - do - if [ $(pgrep ${srvc} | wc -l) -lt "1" ]; then - echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} - touch ${FOLDER}/localhost-${srvc}.down - elif [ -f ${FOLDER}/localhost-${srvc}.down ]; then - echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "${NODE_HOSTNAME}-${nip}" ${STATUS_CHECK_EMAIL} - rm -f ${FOLDER}/localhost-${srvc}.down - fi - done - fi - if [ -z ${LOCAL_SERVICES+x} ] || [ "${2}" = "all" ]; then - nid=1 - for nip in "${NODE_HOSTS[@]}" - do - if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' - else NCMD="ssh root@${nip}" - fi - for srvc in "${NODE_SERVICES[@]}" - do - if [ $(${NCMD} pgrep ${srvc} | wc -l) -lt "1" ]; then - echo "${NM_SERVICES[${srvc}]} is down" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} - touch ${FOLDER}/${nip}-${srvc}.down - elif [ -f ${FOLDER}/${nip}-${srvc}.down ]; then - echo "${NM_SERVICES[${srvc}]} is back UP!" | mail -s "iDS-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} - rm -f ${FOLDER}/${nip}-${srvc}.down - fi - done - nid=`expr $nid + 1` - done - fi - fi -} - SET-PERMISSIONS(){ if [ "${1}" != "" ]; then NEW_SITE=${1}