diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 5bf6601b..28cb967f 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -17,17 +17,21 @@ source /opt/idssys/nodemgmt/defaults.inc # ========================================================= # -STATUS() { - if [ "${1}" = "report" ]; then +STATUS() + start=`date +%s` + ST_ACTION=${1} + if [ "${ST_ACTION}" = "report" ]; then unset idsCL idsBG idsST idsCL=('') idsBG=('') idsST=('') fi declare -i cw; declare -i spc1; declare -i c - if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then - echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}" - echo + if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}" + echo + fi if [ ! -z ${LOCAL_SERVICES+x} ]; then NHCMD='ssh root@10.5.10.51' else NHCMD='' @@ -43,30 +47,41 @@ STATUS() { yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl" fi done - if [ "${1}" = "repl" ]; then + if [ "${ST_ACTION}" = "repl" ]; then sleep 5 fi fi - if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "services" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" + if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsST[Bold]}"; DIVIDER + echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}" + DIVIDER; echo -e "${idsST[Reset]}" + fi if [ ! -z ${LOCAL_SERVICES+x} ]; then - lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) - uptime=`uptime -p` - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" - DIVIDER false green + if [ "${ST_ACTION}" != "check" ]; then + lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) + uptime=`uptime -p` + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" + DIVIDER false green + fi + for srvc in "${LOCAL_SERVICES[@]}"; do - c=0; cw=22; spc='' - spc1=${cw}-${#NM_SERVICES[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${NM_SERVICES[${srvc}]}$spc: " + if [ "${ST_ACTION}" != "check" ]; then + c=0; cw=22; spc='' + spc1=${cw}-${#NM_SERVICES[${srvc}]} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${NM_SERVICES[${srvc}]}$spc: " + fi if [ "$(systemctl is-active ${srvc})" = "active" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi rm -f ${FOLDER}/localhost-${srvc}.down else - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + fi touch ${FOLDER}/localhost-${srvc}.down fi done @@ -77,25 +92,33 @@ STATUS() { if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' else NCMD="ssh root@${nip}"; LH='' fi - uptime=`${NCMD} uptime -p` - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}" - DIVIDER false green + if [ "${ST_ACTION}" != "check" ]; then + uptime=`${NCMD} uptime -p` + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" + DIVIDER false green + fi if [ "${NCMD}" != "" ]; then checkhost=$(CHECK_HOST ${nip}) fi if [ "${checkhost}" != "false" ]; then rm -f ${FOLDER}/${nip}.down for srvc in "${NODE_SERVICES[@]}"; do - c=0; cw=22; spc='' - spc1=${cw}-${#NM_SERVICES[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${NM_SERVICES[${srvc}]}$spc: " + if [ "${ST_ACTION}" != "check" ]; then + c=0; cw=22; spc='' + spc1=${cw}-${#NM_SERVICES[${srvc}]} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${NM_SERVICES[${srvc}]}$spc: " + fi if [ "$(${NCMD} systemctl is-active ${srvc})" = "active" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi rm -f ${FOLDER}/${nip}-${srvc}.down rm -f ${FOLDER}/${nip}-${srvc}.errtime elif [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + fi touch ${FOLDER}/${nip}-${srvc}.down else errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) @@ -104,41 +127,53 @@ STATUS() { mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime fi toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - echo -e "${idsCL[Red]}Not Running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi touch ${FOLDER}/${nip}-${srvc}.down else - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + fi fi fi done else - echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" + fi touch ${FOLDER}/${nip}.down fi - echo + if [ "${ST_ACTION}" != "check" ]; then echo; fi nid=`expr $nid + 1` done fi - if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" + if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsST[Bold]}"; DIVIDER + echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" + DIVIDER; echo -e "${idsST[Reset]}" + fi nid=1 for nip in "${NODE_HOSTS[@]}"; do if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" - DIVIDER false green + if [ "${ST_ACTION}" != "check" ]; then + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" + DIVIDER false green + fi checkhost=$(CHECK_HOST ${nip}) if [ "${checkhost}" != "false" ]; then rm -f ${FOLDER}/${nip}.down for rcheck in "${REPL_CHECK[@]}"; do - c=0; cw=22; spc='' - spc1=${cw}-${#REPL_DESC[${rcheck}]} - timeout=`date --date='10 seconds' +%s` - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${REPL_DESC[${rcheck}]}${spc}: " + if [ "${ST_ACTION}" != "check" ]; then + c=0; cw=22; spc='' + spc1=${cw}-${#REPL_DESC[${rcheck}]} + timeout=`date --date='10 seconds' +%s` + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${REPL_DESC[${rcheck}]}${spc}: " + fi checked=false until [ "${checked}" = "" ]; do if [ "${NHCMD}" = "" ]; then @@ -146,36 +181,45 @@ STATUS() { else checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" fi - #cdate=`date +%s` if [ "`date +%s`" -gt "$timeout" ]; then timeout=true break fi done if [ "${timeout}" != "true" ]; then - echo -e "${idsCL[Green]}Good${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Good${idsCL[Default]}" + fi rm -f ${FOLDER}/${nip}-${rcheck}.down else - echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" + fi touch ${FOLDER}/${nip}-${rcheck}.down - #echo "Timeout occurred in waiting for replication between primary node and iDS-Node${nid} (${nip}) for the replicated folder of '${rcheck}'." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL} fi done else - echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" + fi touch ${FOLDER}/${nip}.down fi - echo + if [ "${ST_ACTION}" != "check" ]; then echo; fi fi nid=`expr $nid + 1` done fi - echo "" - if [ -z $action ] || [ "${action}" = "gui" ]; then - DIVIDER true - ENTER2CONTINUE + if [ "${ST_ACTION}" != "check" ]; then + echo "" + if [ -z $action ] || [ "${action}" = "gui" ]; then + DIVIDER true + ENTER2CONTINUE + fi fi + end=`date +%s` + runtime=$((end-start)) + echo "runtime: ${runtime}" } STATUS-CHECK(){