diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 8647b784..a8741b3b 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -259,90 +259,98 @@ STATUS(){ for NTYPE in "${NODE_TYPES[@]}"; do - PH=${NTYPE}_HOSTS[0] - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - nid=1 + GOREPCHK=false; + if [ date +"%H" -ge "12" ] && [ date +"%H" -le "13" ] && [ "${NTYPE}" = "NC" ]; then + GOREPCHK=true; + elif [ "${NTYPE}" != "NC" ]; then + GOREPCHK=true; + fi + if [ "${GOREPCHK}" = "true" ]; then + PH=${NTYPE}_HOSTS[0] + PH_CMD="ssh root@${!PH}" + var=${NTYPE}_REPL_CHECK[@] + if [ ! -z ${!var+x} ]; then + nid=1 - var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" - DIVIDER false green - fi - checkhost=$(CHECK_HOST ${nip}) - if [ "${checkhost}" != "false" ]; then - if [ -f ${FOLDER}/${nip}.down ]; then - rm -f ${FOLDER}/${nip}.down - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP!" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi - fi - var2=${NTYPE}_REPL_CHECK[@] - # timeout=`date --date='20 seconds' +%s` - for rcheck in "${!var2}"; do - if [ "${ST_ACTION}" != "check" ]; then - c=0; cw=22; spc='' - spc1=${cw}-${#REPL_DESC[${rcheck}]} - # if [ "${rcheck}" = "nextcloud-data" ] || [ "${rcheck}" = "nextcloud-www" ]; then - # timeout=`date --date='2 minutes' +%s` - # else - timeout=`date --date='20 seconds' +%s` - # fi - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${REPL_DESC[${rcheck}]}${spc}: " - else - # if [ "${rcheck}" = "nextcloud-data" ] OR [ "${rcheck}" = "nextcloud-www" ]; then - # timeout=`date --date='5 minutes' +%s` - # else - timeout=`date --date='1 minutes' +%s` - # fi - - fi - checked=false - until [ "${checked}" = "" ]; do - if [ "${PH_CMD}" = "" ]; then - checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` - else - checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" - fi - if [ "`date +%s`" -gt "$timeout" ]; then - timeout=true - break - fi - done - if [ "${timeout}" = "true" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" - fi - if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then - touch ${FOLDER}/${nip}-${rcheck}.down - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Good${idsCL[Default]}" - fi - if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then - rm -f ${FOLDER}/${nip}-${rcheck}.down - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi - fi - fi - - done - else + var=${NTYPE}_HOSTS[@] + for nip in "${!var}"; do + if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" + echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" + DIVIDER false green fi - if [ ! -f ${FOLDER}/${nip}.down ]; then - touch ${FOLDER}/${nip}.down - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}; fi + checkhost=$(CHECK_HOST ${nip}) + if [ "${checkhost}" != "false" ]; then + if [ -f ${FOLDER}/${nip}.down ]; then + rm -f ${FOLDER}/${nip}.down + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP!" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi + fi + var2=${NTYPE}_REPL_CHECK[@] + # timeout=`date --date='20 seconds' +%s` + for rcheck in "${!var2}"; do + if [ "${ST_ACTION}" != "check" ]; then + c=0; cw=22; spc='' + spc1=${cw}-${#REPL_DESC[${rcheck}]} + # if [ "${rcheck}" = "nextcloud-data" ] || [ "${rcheck}" = "nextcloud-www" ]; then + # timeout=`date --date='2 minutes' +%s` + # else + timeout=`date --date='20 seconds' +%s` + # fi + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${REPL_DESC[${rcheck}]}${spc}: " + else + # if [ "${rcheck}" = "nextcloud-data" ] OR [ "${rcheck}" = "nextcloud-www" ]; then + # timeout=`date --date='2 minutes' +%s` + # else + timeout=`date --date='1 minutes' +%s` + # fi + + fi + checked=false + until [ "${checked}" = "" ]; do + if [ "${PH_CMD}" = "" ]; then + checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` + else + checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" + fi + if [ "`date +%s`" -gt "$timeout" ]; then + timeout=true + break + fi + done + if [ "${timeout}" = "true" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" + fi + if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then + touch ${FOLDER}/${nip}-${rcheck}.down + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Good${idsCL[Default]}" + fi + if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then + rm -f ${FOLDER}/${nip}-${rcheck}.down + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi + fi + fi + + done + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" + fi + if [ ! -f ${FOLDER}/${nip}.down ]; then + touch ${FOLDER}/${nip}.down + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}; fi + fi fi + if [ "${ST_ACTION}" != "check" ]; then echo; fi fi - if [ "${ST_ACTION}" != "check" ]; then echo; fi - fi - nid=`expr $nid + 1` - done + nid=`expr $nid + 1` + done + fi fi done fi