diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index fc995d1a..6d9e66e8 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -68,7 +68,7 @@ STATUS(){ fi fi ######################## - if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then + if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then if [ ! -z ${LOCAL_SERVICES+x} ]; then if [ "${ST_ACTION}" != "check" ]; then lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) @@ -133,7 +133,11 @@ STATUS(){ for NTYPE in "${NODE_TYPES[@]}"; do if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" + if [ "${ST_ACTION}" = "dockers" ]; then + echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}" + else + echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" + fi DIVIDER; echo -e "${idsST[Reset]}" fi nid=1 @@ -142,194 +146,208 @@ STATUS(){ hosts=${NTYPE}_HOSTS[@] var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh root@${nip}"; LH='' - fi - if [ "${NCMD}" != "" ]; then - checkhost=$(CHECK_HOST ${nip}) - fi - if [ "${checkhost}" != "false" ]; then - if [ "${ST_ACTION}" != "check" ]; then - uptime=`${NCMD} uptime -p` - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" - DIVIDER false green + + if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" = "dockers" ]; then + GOFORCHECK=false; + else + GOFORCHECK=true; + fi + + if [ ${GOFORCHECK} = true ]; then + for nip in "${!var}"; do + if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' + else NCMD="ssh root@${nip}"; LH='' fi - if [ -f ${FOLDER}/${nip}.down ]; then - if [ -f ${FOLDER}/${nip}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) - else - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) - fi - rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime - # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi + if [ "${NCMD}" != "" ]; then + checkhost=$(CHECK_HOST ${nip}) fi - - var2=${NTYPE}_SERVICES_CHECK[@] - for srvc in "${!var2}"; do - NOGOCHK=true; - if [ ${srvc} = "gitea" ] || [ ${srvc} = "pdnsadmin" ] || [ ${srvc} = "pdnsadmin.socket" ]; then - if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.10.1.120* ]]; then - NOGOCHK=false; - fi - fi - if [ ${NOGOCHK} = true ]; then - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=30; spc='' - spc1=${cw}-${#NM_SERVICES[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${NM_SERVICES[${srvc}]}$spc: " - fi - srvctst=$(${NCMD} systemctl is-active ${srvc}) - #echo "${NCMD} systemctl is-active ${srvc} = ${srvctst}" - #if [ "${srvctst}" = "active" ]; then echo "yes"; else echo "no"; fi - if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then - mysqlgo=false; - [ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true; - elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then - mysqlgo=false; - [ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true; - elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then - mysqlgo=true - fi - #echo "mysqlgo=${mysqlgo}" - if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then - - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - fi - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - touch ${FOLDER}/${nip}-${srvc}.down - # fi - else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime - fi - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - touch ${FOLDER}/${nip}-${srvc}.down - if [ "${srvc}" = "mysql" ]; then - ${NCMD} systemctl restart ${srvc} - fi - fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then - if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) - else - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) - fi - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - - if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then - rm -f ${FOLDER}/*-${srvc}.down - rm -f ${FOLDER}/*-${srvc}.errtime - else - rm -f ${FOLDER}/${nip}-${srvc}.down - rm -f ${FOLDER}/${nip}-${srvc}.errtime - fi - fi - fi - fi - done - - if [[ -v ${NTYPE}_DOCKER ]]; then + if [ "${checkhost}" != "false" ]; then if [ "${ST_ACTION}" != "check" ]; then - echo - echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}" - echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}" + uptime=`${NCMD} uptime -p` + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" + DIVIDER false green + fi + if [ -f ${FOLDER}/${nip}.down ]; then + if [ -f ${FOLDER}/${nip}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + else + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + fi + rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime + # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi fi - for docker in "${!dockers}"; do - if [ "${ST_ACTION}" != "check" ]; then - c=0; cw=30; spc='' - spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}` - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${NM_DOCKERS[${docker}]}$spc: " - fi - if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then - # if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then - # ${NCMD} docker rm ${docker} >/dev/null 2>&1 - # fi - # ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - # sleep 10s - if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" + if [ "${ST_ACTION}" != "dockers" ]; then + var2=${NTYPE}_SERVICES_CHECK[@] + for srvc in "${!var2}"; do + NOGOCHK=true; + if [ ${srvc} = "gitea" ] || [ ${srvc} = "pdnsadmin" ] || [ ${srvc} = "pdnsadmin.socket" ]; then + if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.10.1.120* ]]; then + NOGOCHK=false; fi - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - rm -f ${FOLDER}/${nip}-${docker}.down - rm -f ${FOLDER}/${nip}-${docker}.errtime - else + fi + if [ ${NOGOCHK} = true ]; then if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" - - if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then - if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi - touch ${FOLDER}/${nip}-${docker}.down + if [ "${ST_ACTION}" != "report" ]; then + c=0; cw=30; spc='' + spc1=${cw}-${#NM_SERVICES[${srvc}]} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then - mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime + spc=' ' + fi + echo -en " ${NM_SERVICES[${srvc}]}$spc: " + fi + srvctst=$(${NCMD} systemctl is-active ${srvc}) + #echo "${NCMD} systemctl is-active ${srvc} = ${srvctst}" + #if [ "${srvctst}" = "active" ]; then echo "yes"; else echo "no"; fi + if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then + mysqlgo=false; + [ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true; + elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then + mysqlgo=false; + [ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true; + elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then + mysqlgo=true + fi + #echo "mysqlgo=${mysqlgo}" + if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then + + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime) + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + touch ${FOLDER}/${nip}-${srvc}.down + # fi + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" fi - touch ${FOLDER}/${nip}-${docker}.down + # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + touch ${FOLDER}/${nip}-${srvc}.down + if [ "${srvc}" = "mysql" ]; then + ${NCMD} systemctl restart ${srvc} + fi + fi + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi + if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then + if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + else + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + fi + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + + if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then + rm -f ${FOLDER}/*-${srvc}.down + rm -f ${FOLDER}/*-${srvc}.errtime + else + rm -f ${FOLDER}/${nip}-${srvc}.down + rm -f ${FOLDER}/${nip}-${srvc}.errtime fi fi - fi fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - rm -f ${FOLDER}/${nip}-${docker}.down - rm -f ${FOLDER}/${nip}-${docker}.errtime - fi - done - fi - - else - if [ ! -f ${FOLDER}/${nip}.down ]; then - touch ${FOLDER}/${nip}.down - if [ ! -f ${FOLDER}/${nip}.errtime ]; then - touch ${FOLDER}/${nip}.errtime + done fi - # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then - # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} - # fi + + if [[ -v ${NTYPE}_DOCKER ]]; then + if [ "${ST_ACTION}" != "check" ]; then + echo + echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}" + echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}" + fi + + for docker in "${!dockers}"; do + if [ "${ST_ACTION}" != "check" ]; then + c=0; cw=30; spc='' + spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}` + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${NM_DOCKERS[${docker}]}$spc: " + fi + if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then + # if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then + # ${NCMD} docker rm ${docker} >/dev/null 2>&1 + # fi + # ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 + # sleep 10s + if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" + fi + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + rm -f ${FOLDER}/${nip}-${docker}.down + rm -f ${FOLDER}/${nip}-${docker}.errtime + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" + + if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then + if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi + touch ${FOLDER}/${nip}-${docker}.down + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then + mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime) + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + touch ${FOLDER}/${nip}-${docker}.down + fi + fi + + fi + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi + rm -f ${FOLDER}/${nip}-${docker}.down + rm -f ${FOLDER}/${nip}-${docker}.errtime + fi + done + fi + else - errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) - # if [ $errtime -gt ${RENOTIFY} ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then - # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} - # fi + if [ ! -f ${FOLDER}/${nip}.down ]; then + touch ${FOLDER}/${nip}.down + if [ ! -f ${FOLDER}/${nip}.errtime ]; then + touch ${FOLDER}/${nip}.errtime + fi + # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then + # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} + # fi + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) + # if [ $errtime -gt ${RENOTIFY} ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then + # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} + # fi + fi + if [ "${ST_ACTION}" != "check" ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" + fi fi - if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" - fi - fi - if [ "${ST_ACTION}" != "check" ]; then echo; fi - nid=`expr $nid + 1` - done + if [ "${ST_ACTION}" != "check" ]; then echo; fi + nid=`expr $nid + 1` + done + + else #GOFORCHECK + + fi #GOFORCHECK done