diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index c93b6b7e..aa1c6e64 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -27,126 +27,72 @@ STATUS() { echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}" echo declare -i cw; declare -i spc1; declare -i c - if [ ! -z ${LOCAL_SERVICES+x} ]; - then NHCMD='ssh root@10.5.10.51' - else NHCMD='' - fi - # for nip in "${NODE_HOSTS[@]}"; do - # if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi - # if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then - # NCMD="ssh root@${nip}" - # for rcheck in "${REPL_CHECK[@]}"; do - # ${NCMD} touch ${REPL_CHECKS[${rcheck}]}/test.repl - # done - # fi - # done - - for rcheck in "${REPL_CHECK[@]}"; do - ${NHCMD} rm -f ${FOLDER}/test.repl - daterun=`date +%Y-%m-%d-%H-%M-%S` - if [ "${NHCMD}" = "" ]; then - echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl - yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl - else - ${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl" - yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl" + if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then + if [ ! -z ${LOCAL_SERVICES+x} ]; + then NHCMD='ssh root@10.5.10.51' + else NHCMD='' fi - done - - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - if [ ! -z ${LOCAL_SERVICES+x} ]; then - lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) - uptime=`uptime -p` - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" - DIVIDER false green - for srvc in "${LOCAL_SERVICES[@]}" - do - c=0; cw=22; spc='' - spc1=${cw}-${#NM_SERVICES[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${NM_SERVICES[${srvc}]}$spc: " - if [ $(pgrep ${srvc} | wc -l) -gt "0" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - rm -f ${FOLDER}/localhost-${srvc}.down + for rcheck in "${REPL_CHECK[@]}"; do + ${NHCMD} rm -f ${FOLDER}/test.repl + daterun=`date +%Y-%m-%d-%H-%M-%S` + if [ "${NHCMD}" = "" ]; then + echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl + yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl else - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - touch ${FOLDER}/localhost-${srvc}.down + ${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl" + yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl" fi done - echo fi - nid=1 - for nip in "${NODE_HOSTS[@]}"; do - if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh root@${nip}"; LH='' - fi - uptime=`${NCMD} uptime -p` - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}" - DIVIDER false green - checkhost=$(CHECK_HOST ${nip}) - if [ "${checkhost}" != "false" ]; then - rm -f ${FOLDER}/${nip}.down - for srvc in "${NODE_SERVICES[@]}" + + if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "services" ]; then + echo -e "${idsST[Bold]}"; DIVIDER + echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}" + DIVIDER; echo -e "${idsST[Reset]}" + if [ ! -z ${LOCAL_SERVICES+x} ]; then + lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) + uptime=`uptime -p` + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" + DIVIDER false green + for srvc in "${LOCAL_SERVICES[@]}" do c=0; cw=22; spc='' spc1=${cw}-${#NM_SERVICES[${srvc}]} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en " ${NM_SERVICES[${srvc}]}$spc: " - if [ $(${NCMD} pgrep ${srvc} | wc -l) -gt "0" ]; then + if [ $(pgrep ${srvc} | wc -l) -gt "0" ]; then echo -e "${idsCL[Green]}Running${idsCL[Default]}" - rm -f ${FOLDER}/${nip}-${srvc}.down + rm -f ${FOLDER}/localhost-${srvc}.down else echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - touch ${FOLDER}/${nip}-${srvc}.down + touch ${FOLDER}/localhost-${srvc}.down fi done - else - echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" - touch ${FOLDER}/${nip}.down + echo fi - echo - nid=`expr $nid + 1` - done - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - nid=1 - for nip in "${NODE_HOSTS[@]}"; do - if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi - if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" + nid=1 + for nip in "${NODE_HOSTS[@]}"; do + if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' + else NCMD="ssh root@${nip}"; LH='' + fi + uptime=`${NCMD} uptime -p` + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}" DIVIDER false green checkhost=$(CHECK_HOST ${nip}) if [ "${checkhost}" != "false" ]; then rm -f ${FOLDER}/${nip}.down - for rcheck in "${REPL_CHECK[@]}"; do + for srvc in "${NODE_SERVICES[@]}" + do c=0; cw=22; spc='' - spc1=${cw}-${#REPL_DESC[${rcheck}]} - timeout=`date --date='5 seconds' +%s` + spc1=${cw}-${#NM_SERVICES[${srvc}]} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${REPL_DESC[${rcheck}]}${spc}: " - checked=false - until [ "${checked}" = "" ]; do - if [ "${NHCMD}" = "" ]; then - checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` - else - checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" - fi - #cdate=`date +%s` - if [ "`date +%s`" -gt "$timeout" ]; then - timeout=true - break - fi - done - if [ "${timeout}" != "true" ]; then - echo -e "${idsCL[Green]}Good${idsCL[Default]}" - rm -f ${FOLDER}/${nip}-${rcheck}.down + echo -en " ${NM_SERVICES[${srvc}]}$spc: " + if [ $(${NCMD} pgrep ${srvc} | wc -l) -gt "0" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + rm -f ${FOLDER}/${nip}-${srvc}.down else - echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" - touch ${FOLDER}/${nip}-${rcheck}.down - #echo "Timeout occurred in waiting for replication between primary node and iDS-Node${nid} (${nip}) for the replicated folder of '${rcheck}'." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL} + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + touch ${FOLDER}/${nip}-${srvc}.down fi done else @@ -154,9 +100,60 @@ STATUS() { touch ${FOLDER}/${nip}.down fi echo - fi - nid=`expr $nid + 1` - done + nid=`expr $nid + 1` + done + fi + + if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then + echo -e "${idsST[Bold]}"; DIVIDER + echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" + DIVIDER; echo -e "${idsST[Reset]}" + nid=1 + for nip in "${NODE_HOSTS[@]}"; do + if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi + if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" + DIVIDER false green + checkhost=$(CHECK_HOST ${nip}) + if [ "${checkhost}" != "false" ]; then + rm -f ${FOLDER}/${nip}.down + for rcheck in "${REPL_CHECK[@]}"; do + c=0; cw=22; spc='' + spc1=${cw}-${#REPL_DESC[${rcheck}]} + timeout=`date --date='5 seconds' +%s` + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${REPL_DESC[${rcheck}]}${spc}: " + checked=false + until [ "${checked}" = "" ]; do + if [ "${NHCMD}" = "" ]; then + checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` + else + checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" + fi + #cdate=`date +%s` + if [ "`date +%s`" -gt "$timeout" ]; then + timeout=true + break + fi + done + if [ "${timeout}" != "true" ]; then + echo -e "${idsCL[Green]}Good${idsCL[Default]}" + rm -f ${FOLDER}/${nip}-${rcheck}.down + else + echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" + touch ${FOLDER}/${nip}-${rcheck}.down + #echo "Timeout occurred in waiting for replication between primary node and iDS-Node${nid} (${nip}) for the replicated folder of '${rcheck}'." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL} + fi + done + else + echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" + touch ${FOLDER}/${nip}.down + fi + echo + fi + nid=`expr $nid + 1` + done + fi echo "" if [ -z $action ] || [ "${action}" = "gui" ]; then