diff --git a/inc/status.inc b/inc/status.inc index 221b65a6..aeefe87a 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -670,698 +670,6 @@ STATUS(){ echo "runtime: ${runtime}" } - -REPLCHECK(){ - rcheck=${1} - nip=${2} - PH_CMD=${3} - if [ "${4}" != "check" ]; then - timeout=`date --date='1 minutes' +%s` - else - timeout=`date --date='2 minutes' +%s` - fi - checked=false - until [ "${checked}" == "" ]; do - if [ "${PH_CMD}" == "" ]; then - ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` - else - ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" - fi - [ "`date +%s`" -gt "${timeout}" ] && timeout=true && break - done - [ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good -}#!/usr/bin/env bash - -STATUS(){ - start=`date +%s` - ST_ACTION=${1} - if [ "${ST_ACTION}" = "report" ]; then - unset idsCL idsBG idsST - idsCL=('') - idsBG=('') - idsST=('') - elif [ "${ST_ACTION}" = "sync" ]; then - ST_ACTION=repl - fi - declare -i cw; declare -i spc1; declare -i c - - ######################## - if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}" - fi - for NTYPE in "${NODE_TYPES[@]}"; do - PH=${NTYPE}_HOSTS[0] - if [ ! -f ${FOLDER}/${!PH}.down ]; then - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - for rcheck in "${!var}"; do - # echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}" - ${PH_CMD} rm -f ${FOLDER}/test.repl - daterun=`date +%Y-%m-%d-%H-%M-%S` - ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" & - done - fi - fi - done - # replstart=`date +%s` - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Complete${idsCL[Default]}" - echo - fi - fi - - ######################## - if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then - if [ ! -z ${LOCAL_SERVICES+x} ]; then - if [ "${ST_ACTION}" != "check" ]; then - lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) - uptime=`uptime -p` - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" - DIVIDER false green - fi - for srvc in "${LOCAL_SERVICES[@]}"; do - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=30; spc='' - spc1=${cw}-${#NM_SERVICES[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${NM_SERVICES[${srvc}]}${spc}: " - fi - - if [ "$(systemctl is-active ${srvc})" != "active" ]; then - if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - fi - SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1 - touch ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} - else - errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then - mv ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime - fi - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - touch ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down - fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then - if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) - else - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.down) - fi - rm -f ${TMPFOLDER}/${NODE_HOSTNAME}-${srvc}.* - SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE} - - fi - fi - done - - if [ "${ST_ACTION}" != "check" ]; then echo; fi - fi - ######################## - - for NTYPE in "${NODE_TYPES[@]}"; do - nid=1 - - dockers=${NTYPE}_DOCKER[@] - hosts=${NTYPE}_HOSTS[@] - - var=${NTYPE}_HOSTS[@] - - if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" == "dockers" ]; then - GOFORCHECK=false; - else - GOFORCHECK=true; - fi - - if [ ${GOFORCHECK} = true ]; then - - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - if [ "${ST_ACTION}" = "dockers" ]; then - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}" - else - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" - fi - DIVIDER; echo -e "${idsST[Reset]}" - fi - for nip in "${!var}"; do - [ "${ST_ACTION}" != "check" ] && echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}" - - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh root@${nip}"; LH='' - fi - if [ "${NCMD}" != "" ]; then - checkhost=$(CHECK_HOST ${nip}) - fi - if [ "${checkhost}" != "false" ]; then - if [ "${ST_ACTION}" != "check" ]; then - #echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}" - if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then - uptime=`${NCMD} uptime -p` - echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" - else - echo -e "${idsCL[Default]}" - fi - - DIVIDER false green - fi - if [ -f ${TMPFOLDER}/${nip}.down ]; then - if [ -f ${TMPFOLDER}/${nip}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) - else - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) - fi - rm -f ${TMPFOLDER}/${nip}.* - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - fi - - if [ "${ST_ACTION}" != "dockers" ]; then - srvcs=${NTYPE}_SERVICES_CHECK[@]; - srvcstotest="$(join_by " " ${!srvcs})" - srvctst=(`${NCMD} systemctl is-active ${srvcstotest}`) - sr=0 - for srvc in "${!srvcs}"; do - [ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false || NOGOCHK=true - if [ ${NOGOCHK} == true ]; then - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=30; spc=''; spc1=${cw}-${#NM_SERVICES[${srvc}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${NM_SERVICES[${srvc}]}$spc: " - fi - if [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" == "mysql" ]; then - [ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false - elif [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" = "mariadb" ]; then - [ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false - elif [ "${srvctst[$sr]}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then - mysqlgo=true - fi - # echo "mysqlgo=${mysqlgo}" - if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - touch ${TMPFOLDER}/${nip}-${srvc}.down - # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} - else - errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - [ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}-${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) - touch ${TMPFOLDER}/${nip}-${srvc}.down - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 - # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & - - elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ]; then - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 - - fi - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - else - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" - if [ -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then - [ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} - - rm -f ${TMPFOLDER}/${nip}-${srvc}.* - - fi - # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then - if [ "${srvc}" == "gitea" ]; then - rm -f ${FOLDER}/*-${srvc}.* - fi - fi - [ "${mysqlgo}" = "true" ] && unset mysqlgo - fi - sr=`expr $sr + 1` - done - - - - - - # for srvc in "${!var2}"; do - # NOGOCHK=true; - # [ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false; - # if [ ${NOGOCHK} = true ]; then - # if [ "${ST_ACTION}" != "check" ]; then - # if [ "${ST_ACTION}" != "report" ]; then - # c=0; cw=30; spc='' - # spc1=${cw}-${#NM_SERVICES[${srvc}]} - # until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - # else - # spc=' ' - # fi - # echo -en " ${NM_SERVICES[${srvc}]}$spc: " - # fi - # srvctst=$(${NCMD} systemctl is-active ${srvc}) - # if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then - # mysqlgo=false; - # [ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true; - # elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then - # mysqlgo=false; - # [ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true; - # elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then - # mysqlgo=true - # fi - # # echo "mysqlgo=${mysqlgo}" - # if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - # if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then - # if [ "${ST_ACTION}" != "check" ]; then - # echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - # fi - # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 - # touch ${TMPFOLDER}/${nip}-${srvc}.down - # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} - # - # else - # errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) - # if [ $errtime -gt ${RENOTIFY} ]; then - # if [ ! -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then - # mv ${TMPFOLDER}/${nip}-${srvc}.down ${TMPFOLDER}/${nip}-${srvc}.errtime - # fi - # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) - # touch ${TMPFOLDER}/${nip}-${srvc}.down - # if [ "${srvc}" = "mysql" ]; then - # ${NCMD} systemctl restart ${srvc} - # fi - # fi - # if [ "${ST_ACTION}" != "check" ]; then - # echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - # fi - # fi - # else - # if [ "${ST_ACTION}" != "check" ]; then - # echo -e "${idsCL[Green]}Running${idsCL[Default]}" - # fi - # if [ -f ${TMPFOLDER}/${nip}-${srvc}.down ]; then - # if [ -f ${TMPFOLDER}/${nip}-${srvc}.errtime ]; then - # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.errtime) - # else - # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${srvc}.down) - # fi - # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - # echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} - # - # rm -f ${TMPFOLDER}/${nip}-${srvc}.down - # rm -f ${TMPFOLDER}/${nip}-${srvc}.errtime - # - # fi - # # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then - # if [ "${srvc}" == "gitea" ]; then - # rm -f ${FOLDER}/*-${srvc}.down - # rm -f ${FOLDER}/*-${srvc}.errtime - # fi - # fi - # [ "${mysqlgo}" = "true" ] && unset mysqlgo - # fi - # done - fi - - if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo - echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}" - echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}" - fi - - for docker in "${!dockers}"; do - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=30; spc='' - spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}` - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${NM_DOCKERS[${docker}]}$spc: " - fi - - if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then - ${NCMD} docker rm ${docker} >/dev/null 2>&1 - fi - ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - sleep 10s - if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" - fi - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed" - rm -f ${TMPFOLDER}/${nip}-${docker}.down - rm -f ${TMPFOLDER}/${nip}-${docker}.errtime - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE} - - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" - - if [ ! -f ${TMPFOLDER}/${nip}-${docker}.down ]; then - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1 - touch ${TMPFOLDER}/${nip}-${docker}.down - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE} - else - errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${docker}.down) - if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${TMPFOLDER}/${nip}-${docker}.errtime ]; then - mv ${TMPFOLDER}/${nip}-${docker}.down ${TMPFOLDER}/${nip}-${docker}.errtime - fi - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}-${docker}.errtime) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - touch ${TMPFOLDER}/${nip}-${docker}.down - fi - fi - - fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - rm -f ${TMPFOLDER}/${nip}-${docker}.down - rm -f ${TMPFOLDER}/${nip}-${docker}.errtime - fi - done - fi - - else - if [ ! -f ${TMPFOLDER}/${nip}.down ]; then - touch ${TMPFOLDER}/${nip}.down - if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then - touch ${TMPFOLDER}/${nip}.errtime - fi - if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) - echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" - fi - # SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1 - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE} - else - if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) - echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" - fi - errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) - if [ $errtime -gt 60 ] && [ $errtime -lt 180 ]; then - SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 - fi - # if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then - # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE} - # fi - fi - - fi - if [ "${ST_ACTION}" != "check" ]; then echo; fi - nid=`expr $nid + 1` - done - - fi #GOFORCHECK - - done - - fi - ######################## - ## REPLICATION CHECK - ######################## - - if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - echo -en " ${idsCL[LightCyan]}Starting processes to collect/monitor replication status : " - fi - for NTYPE in "${NODE_TYPES[@]}"; do - PH=${NTYPE}_HOSTS[0] - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then - if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then - var2=${NTYPE}_REPL_CHECK[@] - for rcheck in "${!var2}"; do - REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1 - done - fi - fi - done - fi - done - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - - for NTYPE in "${NODE_TYPES[@]}"; do - PH=${NTYPE}_HOSTS[0] - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" - DIVIDER false green - fi - if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then - var2=${NTYPE}_REPL_CHECK[@] - for rcheck in "${!var2}"; do - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${REPL_DESC[${rcheck}]}${spc}: " - - fi - - checked="" - until [ "${checked}" != "" ]; do - if [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then - checked=good - elif [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then - checked=timeout - fi - rm -f ${TMPFOLDER}/repl.${rcheck}.${nip}.* - done - - if [ "${checked}" == "timeout" ]; then - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" - if [ ! -f ${TMPFOLDER}/${nip}-${rcheck}.down ]; then - touch ${TMPFOLDER}/${nip}-${rcheck}.down - SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1 - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} - fi - else - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}" - if [ -f ${TMPFOLDER}/${nip}-${rcheck}.down ]; then - rm -f ${TMPFOLDER}/${nip}-${rcheck}.down - SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" - echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} - fi - fi - done - else - if [ "${ST_ACTION}" != "check" ]; then - [ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}" - [ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}" - fi - fi - if [ "${ST_ACTION}" != "check" ]; then echo; fi - fi - done - fi - done - ########################## - # REMOVE REPL CHECK FILES - ########################## - if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}" - fi - #NHCMD="ssh root@${MYSQL_HOSTS[0]}" - #LBHCMD="ssh root@${LB_HOSTS[0]}" - #WHCMD="ssh root@${WEB_HOSTS[0]}" - for NTYPE in "${NODE_TYPES[@]}"; do - PH=${NTYPE}_HOSTS[0] - if [ ! -f ${FOLDER}/${!PH}.down ]; then - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - for rcheck in "${!var}"; do - ${PH_CMD} rm -f ${FOLDER}/test.repl - daterun=`date +%Y-%m-%d-%H-%M-%S` - if [ "${PH_CMD}" = "" ]; then - rm -f ${FOLDER}/test.repl & - rm -f ${REPL_CHECKS[${rcheck}]}/test.repl & - else - ${PH_CMD} rm -f ${FOLDER}/test.repl & - ${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl & - fi - done - fi - fi - done - rm -Rf ${FOLDER}/test.repl - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Complete${idsCL[Default]}" - echo - fi - fi - fi - - ######################## - ## FREE SPACE CHECK - ######################## - #if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then - if [ "${ST_ACTION}" = "freespace" ]; then - if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then - for NTYPE in "${NODE_TYPES[@]}"; do - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Free Space Scan${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - fi - nid=1 - var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh root@${nip}"; LH='' - fi - if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}" - DIVIDER false green - fi - - [ "${ST_ACTION}" != "check" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}" - declare -A partitions - DRIVEINFO=$(ssh root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }') - DRIVEINFO=(${DRIVEINFO}) - echo -en "\e[1A"; - echo -e "\e[0K\r" - - NUMDRIVES=$((${#DRIVEINFO[@]} / 3)) - for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do - ii=$((${i}*3)) - pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1 - pname=${pname#*vg-} - if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then - freespace=${DRIVEINFO[$((${ii}+2))]//M/} - if [ "${ST_ACTION}" != "check" ]; then - c=0; cw=20; spc='' - spc1=${cw}-${#pname} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${pname}$spc: " - fi - if [[ ${freespace} -gt 1024 ]]; then - fsgb=$(bc <<< "scale=2; ${freespace}/1024") - fsdsp="${fsgb} GB" - else - fsdsp="${freespace} MB" - fi - if [ "${freespace}" -le "1024" ]; then - fs_status='error' - fs_status_color='Red' - SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free" 1 - elif [ "${freespace}" -le "5120" ]; then - fs_status='warn' - fs_status_color='Yellow' - SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${fsdsp} free" - else - fs_status='' - fs_status_color='Green' - fi - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" - fi - done - - # for partition in "${partitions3[@]}"; do - # if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then - # pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1 - # pname=${pname#*vg-} - # if [ "${ST_ACTION}" != "check" ]; then - # c=0; cw=20; spc='' - # spc1=${cw}-${#pname} - # until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - # echo -en " ${pname}$spc: " - # fi - # - # # [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition} - # # freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1 - # - # if [ "${freespace}" -le "1000" ]; then - # fs_status='error' - # fs_status_color='Red' - # SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" 1 - # - # elif [ "${freespace}" -le "5000" ]; then - # fs_status='warn' - # fs_status_color='Yellow' - # SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" - # - # else - # fs_status='' - # fs_status_color='Green' - # - # fi - # if [ "${ST_ACTION}" != "check" ]; then - # if [[ ${freespace} -gt 1000 ]]; then - # fsgb=$(bc <<< "scale=2; ${freespace}/1000") - # fsdsp="${fsgb} GB" - # else - # fsdsp="${freespace} MB" - # fi - # echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" - # - # fi - # fi - # done - - nid=`expr $nid + 1` - echo - done - done - fi - fi - ######################## - ######################## - - if [ "${ST_ACTION}" != "check" ]; then - echo "" - if [ -z $action ] || [ "${action}" = "gui" ]; then - DIVIDER true - ENTER2CONTINUE - fi - fi - end=`date +%s` - runtime=$((end-start)) - echo "runtime: ${runtime}" -} - REPLCHECK(){ rcheck=${1} nip=${2}