diff --git a/defaults.inc b/defaults.inc index ce2fe9b5..545cf5c4 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='4.17.113-12062023' -NM_BETA=false +VERS='5.1.0a-12062023' +NM_BETA=true noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck ' CERT_DAEMON='/snap/bin/certbot' diff --git a/inc/status.inc b/inc/status.inc index fdf0c729..da24ed7f 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -30,28 +30,16 @@ STATUS(){ echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}" fi for NTYPE in "${ntypesel[@]}"; do - PH=${NTYPE}_HOSTS[0] - if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - for rcheck in "${!var}"; do - # echo "CHECK: ${NTYPE} - ${!PH} - $rcheck - ${NM_REPL_CHECK_LOC[${rcheck}]}" - ${PH_CMD} rm -f ${NM_FOLDER}/test.repl - daterun=`date +%Y-%m-%d-%H-%M-%S` - ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" & - ${PH_CMD} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' - done - fi - fi + REPLSTART ${NTYPE} & done - # replstart=`date +%s` if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}" echo fi fi + ######################## + ## LOCALHOST CHECK ######################## if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then cpu_usage=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') @@ -129,334 +117,27 @@ STATUS(){ if [ "${ST_ACTION}" != "check" ]; then echo; fi fi - ######################## - for NTYPE in "${ntypesel[@]}"; do - - dockers=${NTYPE}_DOCKERS_CHECK[@] - hosts=${NTYPE}_HOSTS[@] - - var=${NTYPE}_HOSTS[@] - - if ([[ ! -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NTYPE}]}" == "" ] && [ "${ST_ACTION}" == "services" ]); then - GOFORCHECK=false; - else - GOFORCHECK=true; - fi + ################################### + ## NODE SERVICE AND DOCKER CHECK + ################################### - if [ ${GOFORCHECK} = true ]; then - - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - if [ "${ST_ACTION}" == "dockers" ]; then - echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}" - else - echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" - fi - DIVIDER . lightYellow; echo -e "${idsST[Reset]}" + for NTYPE in "${ntypesel[@]}"; do + STATUS_NODE ${NTYPE} > ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.running & + done + for NTYPE in "${ntypesel[@]}"; do + completed=false + until [ "${completed}" == "true" ]; do + if [ -f ${NM_TMPFOLDER}/status-check.${NTYPE}.done ]; then + completed=true + echo -e "$(cat ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.running)" + rm -f ${NM_TMPFOLDER}/status-check.${NTYPE}.* fi - for nip in "${!var}"; do - if [ "${ST_ACTION}" != "check" ]; then - echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}" - [ "${ST_ACTION}" != "report" ] && echo -en " - Verifying Host..." - fi - - if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then - NCMD=''; LH='- localhost' - else - NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH='' - fi - if [ "${NCMD}" != "" ]; then - checkhost=$(CHECK_HOST ${nip}) - [ "${checkhost}" != "false" ] && checkhostssl=$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1) - else - checkhost=true - checkhostssl=ok - fi - # echo "HERE: $checkhost - $checkhostssl" - if [ "${checkhost}" != "false" ] && [ "${checkhostssl}" == "ok" ]; then - [ "${NTYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage="" - - - if [ "${ST_ACTION}" != "check" ]; then - if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then - if [ "${NTYPE}" == "OFW" ]; then - uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1) - if [ ${#uptime} -ge 6 ]; then - uptime="up $(echo $uptime | cut -d: -f1) days, $(echo $uptime | cut -d: -f2) hours, $(echo $uptime | cut -d: -f3) minutes" - elif [ ${#uptime} -ge 3 ]; then - uptime="up $(echo $uptime | cut -d: -f1) hours, $(echo $uptime | cut -d: -f2) minutes" - else - uptime="up ${uptime} minutes" - fi - - else - uptime=$(${NCMD} uptime -p) - fi - [ "${ST_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[White]} - ${uptime} - ${idsCL[Yellow]}CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%${idsCL[Default]}" - else - echo -e "${idsCL[Default]}" - fi - - DIVIDER false lightCyan - fi - - if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then - if [ -f ${NM_TMPFOLDER}/${nip}.sent ]; then - [ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" - fi - rm -f ${NM_TMPFOLDER}/${nip}.* - fi - - ######################## - ## SERVICES CHECK - ######################## - if [ "${ST_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NTYPE}]}" != "" ]; then - echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}" - DIVIDER . green 52 - - srvcs=${NTYPE}_SERVICES_CHECK[@]; - srvcstotest="$(join_by " " ${!srvcs})" - [ "${NTYPE}" != "OFW" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}"`) - sr=0 - for srvc in "${!srvcs}"; do - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=${PRI_CW}; spc='' - spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - tmp=${NM_SINGLESRVR_SERVICES[${NTYPE}]} - if [[ ${tmp[@]} =~ ${srvc} ]]; then - if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NTYPE}]}* ]]; then - SSCHK=false - rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* - else - SSCHK=primary - fi - else - SSCHK=true - fi - - [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]) && SSCHK=false - - if [ ${SSCHK} != false ]; then - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: " - if [ "${NTYPE}" != "OFW" ]; then - srvctest=${srvctst[$sr]} - if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then - [ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false - elif [ "${srvctest}" != "active" ] && [ "${srvc}" = "mariadb" ]; then - [ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false - elif [ "${srvctest}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then - mysqlgo=true - fi - else - if [ "$(ssh -o ConnectTimeout=3 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then - srvctest=active - else - srvctest=notactive - fi - fi - if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running" - touch ${NM_TMPFOLDER}/${nip}~${srvc}.down - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} - [ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} & - - elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then - errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) - if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then - [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) - touch ${NM_TMPFOLDER}/${nip}~${srvc}.down - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & - - elif [ ${errtime} -gt 60 ] && [ ${errtime} -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - - fi - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})" - fi - else - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running" - if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then - [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} - - rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* - - fi - # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then - if [ "${srvc}" == "gitea" ] || [ "${srvc}" == "headscale" ]; then - rm -f ${NM_FOLDER}/*~${srvc}.* - fi - fi - [ "${mysqlgo}" = "true" ] && unset mysqlgo - [ "${SSCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}" - - elif [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]); then - echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled${idsCL[Default]}" - - else - echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}" - - fi - sr=`expr $sr + 1` - done - - fi - - ######################## - ## DOCKER CHECK - ######################## - if [[ -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" != "services" ]; then - if [ "${ST_ACTION}" != "check" ]; then - [ "${ST_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NTYPE}]}" != "" ] && echo - echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}" - DIVIDER . green 52 - fi - for docker in "${!dockers}"; do - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=${PRI_CW}; spc='' - spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}` - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - tmp=${NM_SINGLESRVR_DOCKERS[${NTYPE}]} - if [[ ${tmp[@]} =~ ${docker} ]]; then - if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NTYPE}]}* ]]; then - SDCHK=false - rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* - else - SDCHK=primary - fi - else - SDCHK=true - fi - if [ ${SDCHK} != false ]; then - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: " - - if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then - ${NCMD} docker start ${docker} >/dev/null 2>&1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} - else - ${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} - fi - sleep 10s - if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[Green]}Running - Fixed" - fi - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed" - rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} - - else - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[Red]}Not Running - Could Not Fix!" - - if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1 - touch ${NM_TMPFOLDER}/${nip}~${docker}.down - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} - else - errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) - if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then - if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then - mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime - fi - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - touch ${NM_TMPFOLDER}/${nip}~${docker}.down - fi - fi - - fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[LightGreen]}Running" - fi - rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* - fi - [ "${SDCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}" - else - echo -e "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}" - fi - done - fi - - else - [ "${ST_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}" - rm -f ${NM_TMPFOLDER}/${nip}~* - if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then - touch ${NM_TMPFOLDER}/${nip}.down - # [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && touch ${NM_TMPFOLDER}/${nip}.errtime - if [ "${ST_ACTION}" != "check" ]; then - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) - if [ "${checkhost}" != "false" ]; then - echo -e "${idsCL[Red]} - Node is online, but SSL is down!${idsCL[Default]}" - else - echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" - fi - fi - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 - [ "${checkhostssl}" != "ok" ] && echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE} - else - if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then - mv touch ${NM_TMPFOLDER}/${nip}.down touch ${NM_TMPFOLDER}/${nip}.errtime - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) - errtime=${toterrtime} - if [ "${checkhost}" != "false" ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is online, but SSL is down" 1 - else - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 - fi - touch ${NM_TMPFOLDER}/${nip}.sent - else - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) - errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) - fi - - if [ "${ST_ACTION}" != "check" ]; then - if [ "${checkhost}" != "false" ]; then - echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" - else - echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" - fi - fi - if [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then - if [ "${checkhost}" != "false" ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSL has been down for $(SHOW_TIME ${toterrtime})" 1 - else - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1 - fi - touch ${NM_TMPFOLDER}/${nip}.sent - fi - touch ${NM_TMPFOLDER}/${nip}.down - fi - - fi - [ "${ST_ACTION}" != "check" ] && echo - done - - fi #GOFORCHECK - + done done fi + ######################## ## REPLICATION CHECK ######################## @@ -497,7 +178,7 @@ STATUS(){ for nip in "${!var}"; do if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}" + echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]}[${!PH}] <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}" DIVIDER false green fi if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then @@ -515,9 +196,9 @@ STATUS(){ checked="" until [ "${checked}" != "" ]; do - if [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then + if [ -f ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then checked=good - elif [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then + elif [ -f ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then checked=timeout fi done @@ -588,8 +269,6 @@ It was down for $(SHOW_TIME ${toterrtime})" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then for rcheck in "${!var}"; do - ${PH_CMD} rm -f ${NM_FOLDER}/test.repl - daterun=`date +%Y-%m-%d-%H-%M-%S` if [ "${PH_CMD}" = "" ]; then rm -f ${NM_FOLDER}/test.repl & rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl & @@ -597,16 +276,12 @@ It was down for $(SHOW_TIME ${toterrtime})" ${PH_CMD} rm -f ${NM_FOLDER}/test.repl & ${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl & fi - rm -f ${NM_TMPFOLDER}/repl.${rcheck}.* done fi fi done - rm -Rf ${NM_FOLDER}/test.repl - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}" - echo - fi + rm -f ${NM_FOLDER}/test.repl + [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}\n" fi fi @@ -738,6 +413,332 @@ It was down for $(SHOW_TIME ${toterrtime})" [ "${ST_ACTION}" == "report" ] && echo ${runtime} >> ${NM_LOGFOLDER}/status-check.scantimes } +STATUS_NODE(){ + NODETYPE=${1} + dockers=${NODETYPE}_DOCKERS_CHECK[@] + hosts=${NODETYPE}_HOSTS[@] + + var=${NODETYPE}_HOSTS[@] + + if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${ST_ACTION}" == "services" ]); then + GOFORCHECK=false; + else + GOFORCHECK=true; + fi + + if [ ${GOFORCHECK} = true ]; then + + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsST[Bold]}"; DIVIDER + if [ "${ST_ACTION}" == "dockers" ]; then + echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}" + else + echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" + fi + DIVIDER . lightYellow; echo -e "${idsST[Reset]}" + fi + for nip in "${!var}"; do + if [ "${ST_ACTION}" != "check" ]; then + echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}" + [ "${ST_ACTION}" != "report" ] && echo -en " - Verifying Host..." + fi + + if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then + NCMD=''; LH='- localhost' + else + NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH='' + fi + if [ "${NCMD}" != "" ]; then + checkhost=$(CHECK_HOST ${nip}) + [ "${checkhost}" != "false" ] && checkhostssl=$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1) + else + checkhost=true + checkhostssl=ok + fi + # echo "HERE: $checkhost - $checkhostssl" + if [ "${checkhost}" != "false" ] && [ "${checkhostssl}" == "ok" ]; then + [ "${NODETYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage="" + + + if [ "${ST_ACTION}" != "check" ]; then + if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then + if [ "${NODETYPE}" == "OFW" ]; then + uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1) + if [ ${#uptime} -ge 6 ]; then + uptime="up $(echo $uptime | cut -d: -f1) days, $(echo $uptime | cut -d: -f2) hours, $(echo $uptime | cut -d: -f3) minutes" + elif [ ${#uptime} -ge 3 ]; then + uptime="up $(echo $uptime | cut -d: -f1) hours, $(echo $uptime | cut -d: -f2) minutes" + else + uptime="up ${uptime} minutes" + fi + + else + uptime=$(${NCMD} uptime -p) + fi + [ "${ST_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[White]} - ${uptime} - ${idsCL[Yellow]}CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%${idsCL[Default]}" + else + echo -e "${idsCL[Default]}" + fi + + DIVIDER false lightCyan + fi + + if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then + if [ -f ${NM_TMPFOLDER}/${nip}.sent ]; then + [ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" + fi + rm -f ${NM_TMPFOLDER}/${nip}.* + fi + + ######################## + ## SERVICES CHECK + ######################## + if [ "${ST_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then + echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}" + DIVIDER . green 52 + + srvcs=${NODETYPE}_SERVICES_CHECK[@]; + srvcstotest="$(join_by " " ${!srvcs})" + [ "${NODETYPE}" != "OFW" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}"`) + sr=0 + for srvc in "${!srvcs}"; do + if [ "${ST_ACTION}" != "report" ]; then + c=0; cw=${PRI_CW}; spc='' + spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + else + spc=' ' + fi + tmp=${NM_SINGLESRVR_SERVICES[${NODETYPE}]} + if [[ ${tmp[@]} =~ ${srvc} ]]; then + if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then + SSCHK=false + rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* + else + SSCHK=primary + fi + else + SSCHK=true + fi + + [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]) && SSCHK=false + + if [ ${SSCHK} != false ]; then + [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: " + if [ "${NODETYPE}" != "OFW" ]; then + srvctest=${srvctst[$sr]} + if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then + [ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false + elif [ "${srvctest}" != "active" ] && [ "${srvc}" = "mariadb" ]; then + [ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false + elif [ "${srvctest}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then + mysqlgo=true + fi + else + if [ "$(ssh -o ConnectTimeout=3 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then + srvctest=active + else + srvctest=notactive + fi + fi + if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then + if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then + [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running" + touch ${NM_TMPFOLDER}/${nip}~${srvc}.down + # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} + [ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} & + + elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then + errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) + if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then + [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime + toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) + touch ${NM_TMPFOLDER}/${nip}~${srvc}.down + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 + # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & + + elif [ ${errtime} -gt 60 ] && [ ${errtime} -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 + + fi + [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})" + fi + else + [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running" + if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then + [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} + + rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* + + fi + # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then + if [ "${srvc}" == "gitea" ] || [ "${srvc}" == "headscale" ]; then + rm -f ${NM_FOLDER}/*~${srvc}.* + fi + fi + [ "${mysqlgo}" = "true" ] && unset mysqlgo + [ "${SSCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}" + + elif [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]); then + echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled${idsCL[Default]}" + + else + echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}" + + fi + sr=`expr $sr + 1` + done + + fi + + ######################## + ## DOCKER CHECK + ######################## + if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" != "services" ]; then + if [ "${ST_ACTION}" != "check" ]; then + [ "${ST_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo + echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}" + DIVIDER . green 52 + fi + for docker in "${!dockers}"; do + if [ "${ST_ACTION}" != "report" ]; then + c=0; cw=${PRI_CW}; spc='' + spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}` + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + else + spc=' ' + fi + tmp=${NM_SINGLESRVR_DOCKERS[${NODETYPE}]} + if [[ ${tmp[@]} =~ ${docker} ]]; then + if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then + SDCHK=false + rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* + else + SDCHK=primary + fi + else + SDCHK=true + fi + if [ ${SDCHK} != false ]; then + [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: " + + if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then + if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then + ${NCMD} docker start ${docker} >/dev/null 2>&1 + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} + else + ${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} + fi + sleep 10s + if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -en "${idsCL[Green]}Running - Fixed" + fi + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed" + rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} + + else + if [ "${ST_ACTION}" != "check" ]; then + echo -en "${idsCL[Red]}Not Running - Could Not Fix!" + + if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1 + touch ${NM_TMPFOLDER}/${nip}~${docker}.down + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} + else + errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) + if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then + if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then + mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime + fi + toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)) + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + touch ${NM_TMPFOLDER}/${nip}~${docker}.down + fi + fi + + fi + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -en "${idsCL[LightGreen]}Running" + fi + rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* + fi + [ "${SDCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}" + else + echo -e "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}" + fi + done + fi + + else + [ "${ST_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}" + rm -f ${NM_TMPFOLDER}/${nip}~* + if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then + touch ${NM_TMPFOLDER}/${nip}.down + # [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && touch ${NM_TMPFOLDER}/${nip}.errtime + if [ "${ST_ACTION}" != "check" ]; then + if [ "${checkhost}" != "false" ]; then + echo -e "${idsCL[Red]} - Node is online, but SSL is down!${idsCL[Default]}" + else + echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" + fi + fi + # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 + [ "${checkhostssl}" != "ok" ] && echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE} + else + if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then + mv ${NM_TMPFOLDER}/${nip}.down ${NM_TMPFOLDER}/${nip}.errtime + toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) + errtime=${toterrtime} + if [ "${checkhost}" != "false" ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is online, but SSL is down" 1 + else + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 + fi + touch ${NM_TMPFOLDER}/${nip}.sent + else + toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) + errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) + fi + + if [ "${ST_ACTION}" != "check" ]; then + if [ "${checkhost}" != "false" ]; then + echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" + else + echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" + fi + fi + if [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then + if [ "${checkhost}" != "false" ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSL has been down for $(SHOW_TIME ${toterrtime})" 1 + else + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1 + fi + touch ${NM_TMPFOLDER}/${nip}.sent + fi + touch ${NM_TMPFOLDER}/${nip}.down + fi + + fi + [ "${ST_ACTION}" != "check" ] && echo + done + + fi #GOFORCHECK + + touch ${NM_TMPFOLDER}/status-check.${NTYPE}.done + +} + REPLCHECK(){ rcheck=${1} nip=${2} @@ -761,7 +762,7 @@ REPLCHECK(){ fi [ "`date +%s`" -gt "${timeout}" ] && timeout=true && break done - [ "${timeout}" == "true" ] && touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good + [ "${timeout}" == "true" ] && touch ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.good } REPLSTART(){ diff --git a/inc/status.new.inc b/inc/status.new.inc deleted file mode 100755 index 6f13a629..00000000 --- a/inc/status.new.inc +++ /dev/null @@ -1,738 +0,0 @@ -#!/usr/bin/env bash -STATUSNEW(){ - start=`date +%s` - - PRI_CW=40; - - ST_ACTION=${1} - if [ "${ST_ACTION}" = "sync" ]; then - ST_ACTION=repl - fi - declare -i cw; declare -i spc1; declare -i c - - ntypesel="" - for NTYPE in "${NM_NODE_TYPES[@]}"; do - if [ "${ST_ACTION^^}" == "${NTYPE}" ]; then - ntypesel=${NTYPE} - ST_ACTION=${2} - break - fi - done - if [ "${ntypesel}" != "" ]; then - ntypesel=(${ntypesel}) - else - ntypesel=(${NM_NODE_TYPES[@]}) - fi - - ######################## - if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}" - fi - for NTYPE in "${ntypesel[@]}"; do - REPLSTART ${NTYPE} & - done - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}" - echo - fi - fi - - ######################## - if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then - cpu_usage=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') - - if [ ! -z ${LOCAL_SERVICES+x} ] && [ "${ST_ACTION}" != "dockers" ]; then - if [ "${ST_ACTION}" != "check" ]; then - lip=${RUN_NODE_IP} - echo -e "\n$(DIVIDER)\n${idsCL[Yellow]}${idsST[Bold]} LOCALHOST Service Status${idsST[Reset]}${idsCL[Default]}" - echo -e "$(DIVIDER)\n" - echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}" - if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then - uptime=`uptime -p` - echo -e "${idsCL[LightCyan]} - ${uptime} - ${idsCL[Yellow]}CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%${idsCL[Default]}" - else - echo -e "${idsCL[Default]}" - fi - DIVIDER . green - fi - # if [ "${ST_ACTION}" != "check" ]; then - # lip=${RUN_NODE_IP} - # uptime=`uptime -p` - # echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" - # DIVIDER false green - # fi - for srvc in "${LOCAL_SERVICES[@]}"; do - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=${PRI_CW}; spc='' - spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: " - fi - - if [ "$(systemctl is-active ${srvc})" != "active" ]; then - if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - fi - SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down - echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} - else - errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)) - if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then - if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then - mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime - fi - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down - fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}" - fi - if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then - if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)) - else - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)) - fi - rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.* - SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE} - - fi - fi - done - - if [ "${ST_ACTION}" != "check" ]; then echo; fi - fi - ######################## - - for NTYPE in "${ntypesel[@]}"; do - STATUS_NODE ${NTYPE} > ${NM_TMPFOLDER}/status-check.${NTYPE}.running & - done - for NTYPE in "${ntypesel[@]}"; do - completed=false - until [ "${completed}" == "true" ]; do - if [ -f ${NM_TMPFOLDER}/status-check.${NTYPE}.done ]; then - completed=true - echo -e "$(cat ${NM_TMPFOLDER}/status-check.${NTYPE}.running)" - rm -f ${NM_TMPFOLDER}/status-check.${NTYPE}.* - fi - done - done - - fi - ######################## - ## REPLICATION CHECK - ######################## - - if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - echo -en " ${idsCL[LightCyan]}Starting processes to collect/monitor replication status : " - fi - for NTYPE in "${ntypesel[@]}"; do - PH=${NTYPE}_HOSTS[0] - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then - if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then - var2=${NTYPE}_REPL_CHECK[@] - for rcheck in "${!var2}"; do - REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1 - done - fi - fi - done - fi - done - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - - for NTYPE in "${ntypesel[@]}"; do - PH=${NTYPE}_HOSTS[0] - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]}[${!PH}] <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}" - DIVIDER false green - fi - if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then - var2=${NTYPE}_REPL_CHECK[@] - for rcheck in "${!var2}"; do - if [ "${ST_ACTION}" != "check" ]; then - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=${PRI_CW}; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - echo -en " ${NM_REPL_DESC[${rcheck}]}${spc}: " - - fi - - checked="" - until [ "${checked}" != "" ]; do - if [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then - checked=good - elif [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then - checked=timeout - fi - done - - if [ "${checked}" == "timeout" ]; then - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" - if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then - touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down - elif [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ]; then - mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime - touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down - SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE} - else - snderrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)) - # echo "HERE: $snderrtime" - if [ ${snderrtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)) - SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]}) -It has been down for $(SHOW_TIME ${toterrtime})" 1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, has now been down for: $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} - touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down - fi - fi - - elif [ "${checked}" == "good" ]; then - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}" - if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then - if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ]; then - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)) - SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]}) -It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE} - fi - rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl* - fi - ssh -q root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' - fi - done - - else - if [ "${ST_ACTION}" != "check" ]; then - [ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}" - [ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}" - fi - fi - [ "${ST_ACTION}" != "check" ] && echo - - fi - - done - fi - done - ########################## - # REMOVE REPL CHECK FILES - ########################## - if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}" - fi - #NHCMD="ssh root@${NM_HOSTS['MYSQL'][0]}" - #LBHCMD="ssh root@${NM_HOSTS['LB'][0]}" - #WHCMD="ssh root@${NM_HOSTS['WEB'][0]}" - for NTYPE in "${ntypesel[@]}"; do - PH=${NTYPE}_HOSTS[0] - if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then - PH_CMD="ssh root@${!PH}" - var=${NTYPE}_REPL_CHECK[@] - if [ ! -z ${!var+x} ]; then - for rcheck in "${!var}"; do - if [ "${PH_CMD}" = "" ]; then - rm -f ${NM_FOLDER}/test.repl & - rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl & - else - ${PH_CMD} rm -f ${NM_FOLDER}/test.repl & - ${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl & - fi - rm -f ${NM_TMPFOLDER}/repl.${rcheck}.* - done - fi - fi - done - rm -Rf ${NM_FOLDER}/test.repl - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}" - echo - fi - fi - fi - - ######################## - ## FREE SPACE CHECK - ######################## - #if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then - if [ "${ST_ACTION}" = "freespace" ]; then - if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then - for NTYPE in "${ntypesel[@]}"; do - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node(s) Free Space Scan${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - fi - var=${NTYPE}_HOSTS[@] - for nip in "${!var}"; do - if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH='' - fi - if [ "${ST_ACTION}" != "check" ]; then - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}" - DIVIDER false green - fi - - [ "${ST_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}" - declare -A partitions - DRIVEINFO=$(ssh -o ConnectTimeout=3 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }') - DRIVEINFO=(${DRIVEINFO}) - if [ "${2}" != "report" ]; then - echo -en "\e[1A" - echo -e "\e[0K\r" - fi - - NUMDRIVES=$((${#DRIVEINFO[@]} / 3)) - for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do - ii=$((${i}*3)) - pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1 - pname=${pname#*vg-} - if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then - freespace=${DRIVEINFO[$((${ii}+2))]//M/} - if [ "${ST_ACTION}" != "check" ]; then - c=0; cw=20; spc='' - spc1=${cw}-${#pname} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${pname}$spc: " - fi - if [[ ${freespace} -gt 1024 ]]; then - fsgb=$(bc <<< "scale=2; ${freespace}/1024") - fsdsp="${fsgb} GB" - else - fsdsp="${freespace} MB" - fi - if [ "${freespace}" -le "1024" ]; then - fs_status='error' - fs_status_color='Red' - SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free" 1 - elif [ "${freespace}" -le "5120" ]; then - fs_status='warn' - fs_status_color='Yellow' - SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free" - else - fs_status='' - fs_status_color='Green' - fi - [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" - fi - done - - # for partition in "${partitions3[@]}"; do - # if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then - # pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1 - # pname=${pname#*vg-} - # if [ "${ST_ACTION}" != "check" ]; then - # c=0; cw=20; spc='' - # spc1=${cw}-${#pname} - # until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - # echo -en " ${pname}$spc: " - # fi - # - # # [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition} - # # freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1 - # - # if [ "${freespace}" -le "1000" ]; then - # fs_status='error' - # fs_status_color='Red' - # SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free" 1 - # - # elif [ "${freespace}" -le "5000" ]; then - # fs_status='warn' - # fs_status_color='Yellow' - # SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free" - # - # else - # fs_status='' - # fs_status_color='Green' - # - # fi - # if [ "${ST_ACTION}" != "check" ]; then - # if [[ ${freespace} -gt 1000 ]]; then - # fsgb=$(bc <<< "scale=2; ${freespace}/1000") - # fsdsp="${fsgb} GB" - # else - # fsdsp="${freespace} MB" - # fi - # echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" - # - # fi - # fi - # done - - echo - done - done - fi - fi - ######################## - ######################## - if [ "${ST_ACTION}" != "check" ]; then - echo - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then - DIVIDER true - ENTER2CONTINUE - fi - fi - end=`date +%s` - runtime=$((end-start)) - echo -e "Runtime: ${runtime}\n" - [ "${ST_ACTION}" == "report" ] && echo ${runtime} >> ${NM_LOGFOLDER}/status-check.scantimes -} - -STATUS_NODE(){ - NODETYPE=${1} - dockers=${NODETYPE}_DOCKERS_CHECK[@] - hosts=${NODETYPE}_HOSTS[@] - - var=${NODETYPE}_HOSTS[@] - - if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${ST_ACTION}" == "services" ]); then - GOFORCHECK=false; - else - GOFORCHECK=true; - fi - - if [ ${GOFORCHECK} = true ]; then - - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - if [ "${ST_ACTION}" == "dockers" ]; then - echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}" - else - echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" - fi - DIVIDER . lightYellow; echo -e "${idsST[Reset]}" - fi - for nip in "${!var}"; do - if [ "${ST_ACTION}" != "check" ]; then - echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}" - [ "${ST_ACTION}" != "report" ] && echo -en " - Verifying Host..." - fi - - if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then - NCMD=''; LH='- localhost' - else - NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH='' - fi - if [ "${NCMD}" != "" ]; then - checkhost=$(CHECK_HOST ${nip}) - [ "${checkhost}" != "false" ] && checkhostssl=$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1) - else - checkhost=true - checkhostssl=ok - fi - # echo "HERE: $checkhost - $checkhostssl" - if [ "${checkhost}" != "false" ] && [ "${checkhostssl}" == "ok" ]; then - [ "${NODETYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage="" - - - if [ "${ST_ACTION}" != "check" ]; then - if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then - if [ "${NODETYPE}" == "OFW" ]; then - uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1) - if [ ${#uptime} -ge 6 ]; then - uptime="up $(echo $uptime | cut -d: -f1) days, $(echo $uptime | cut -d: -f2) hours, $(echo $uptime | cut -d: -f3) minutes" - elif [ ${#uptime} -ge 3 ]; then - uptime="up $(echo $uptime | cut -d: -f1) hours, $(echo $uptime | cut -d: -f2) minutes" - else - uptime="up ${uptime} minutes" - fi - - else - uptime=$(${NCMD} uptime -p) - fi - [ "${ST_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[White]} - ${uptime} - ${idsCL[Yellow]}CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%${idsCL[Default]}" - else - echo -e "${idsCL[Default]}" - fi - - DIVIDER false lightCyan - fi - - if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then - if [ -f ${NM_TMPFOLDER}/${nip}.sent ]; then - [ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" - fi - rm -f ${NM_TMPFOLDER}/${nip}.* - fi - - ######################## - ## SERVICES CHECK - ######################## - if [ "${ST_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then - echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}" - DIVIDER . green 52 - - srvcs=${NODETYPE}_SERVICES_CHECK[@]; - srvcstotest="$(join_by " " ${!srvcs})" - [ "${NODETYPE}" != "OFW" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}"`) - sr=0 - for srvc in "${!srvcs}"; do - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=${PRI_CW}; spc='' - spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]} - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - tmp=${NM_SINGLESRVR_SERVICES[${NODETYPE}]} - if [[ ${tmp[@]} =~ ${srvc} ]]; then - if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then - SSCHK=false - rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* - else - SSCHK=primary - fi - else - SSCHK=true - fi - - [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]) && SSCHK=false - - if [ ${SSCHK} != false ]; then - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: " - if [ "${NODETYPE}" != "OFW" ]; then - srvctest=${srvctst[$sr]} - if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then - [ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false - elif [ "${srvctest}" != "active" ] && [ "${srvc}" = "mariadb" ]; then - [ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false - elif [ "${srvctest}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then - mysqlgo=true - fi - else - if [ "$(ssh -o ConnectTimeout=3 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then - srvctest=active - else - srvctest=notactive - fi - fi - if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running" - touch ${NM_TMPFOLDER}/${nip}~${srvc}.down - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} - [ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} & - - elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then - errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) - if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then - [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) - touch ${NM_TMPFOLDER}/${nip}~${srvc}.down - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & - - elif [ ${errtime} -gt 60 ] && [ ${errtime} -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - - fi - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})" - fi - else - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running" - if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then - [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} - - rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* - - fi - # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then - if [ "${srvc}" == "gitea" ] || [ "${srvc}" == "headscale" ]; then - rm -f ${NM_FOLDER}/*~${srvc}.* - fi - fi - [ "${mysqlgo}" = "true" ] && unset mysqlgo - [ "${SSCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}" - - elif [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]); then - echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled${idsCL[Default]}" - - else - echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}" - - fi - sr=`expr $sr + 1` - done - - fi - - ######################## - ## DOCKER CHECK - ######################## - if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" != "services" ]; then - if [ "${ST_ACTION}" != "check" ]; then - [ "${ST_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo - echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}" - DIVIDER . green 52 - fi - for docker in "${!dockers}"; do - if [ "${ST_ACTION}" != "report" ]; then - c=0; cw=${PRI_CW}; spc='' - spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}` - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - else - spc=' ' - fi - tmp=${NM_SINGLESRVR_DOCKERS[${NODETYPE}]} - if [[ ${tmp[@]} =~ ${docker} ]]; then - if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then - SDCHK=false - rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* - else - SDCHK=primary - fi - else - SDCHK=true - fi - if [ ${SDCHK} != false ]; then - [ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: " - - if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then - ${NCMD} docker start ${docker} >/dev/null 2>&1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} - else - ${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} - fi - sleep 10s - if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[Green]}Running - Fixed" - fi - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed" - rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} - - else - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[Red]}Not Running - Could Not Fix!" - - if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1 - touch ${NM_TMPFOLDER}/${nip}~${docker}.down - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} - else - errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) - if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then - if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then - mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime - fi - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)) - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" - fi - touch ${NM_TMPFOLDER}/${nip}~${docker}.down - fi - fi - - fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -en "${idsCL[LightGreen]}Running" - fi - rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* - fi - [ "${SDCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}" - else - echo -e "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}" - fi - done - fi - - else - [ "${ST_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}" - rm -f ${NM_TMPFOLDER}/${nip}~* - if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then - touch ${NM_TMPFOLDER}/${nip}.down - # [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && touch ${NM_TMPFOLDER}/${nip}.errtime - if [ "${ST_ACTION}" != "check" ]; then - if [ "${checkhost}" != "false" ]; then - echo -e "${idsCL[Red]} - Node is online, but SSL is down!${idsCL[Default]}" - else - echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" - fi - fi - # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 - [ "${checkhostssl}" != "ok" ] && echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE} - else - if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then - mv ${NM_TMPFOLDER}/${nip}.down ${NM_TMPFOLDER}/${nip}.errtime - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) - errtime=${toterrtime} - if [ "${checkhost}" != "false" ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is online, but SSL is down" 1 - else - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 - fi - touch ${NM_TMPFOLDER}/${nip}.sent - else - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) - errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) - fi - - if [ "${ST_ACTION}" != "check" ]; then - if [ "${checkhost}" != "false" ]; then - echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" - else - echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" - fi - fi - if [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then - if [ "${checkhost}" != "false" ]; then - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSL has been down for $(SHOW_TIME ${toterrtime})" 1 - else - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1 - fi - touch ${NM_TMPFOLDER}/${nip}.sent - fi - touch ${NM_TMPFOLDER}/${nip}.down - fi - - fi - [ "${ST_ACTION}" != "check" ] && echo - done - - fi #GOFORCHECK - - touch ${NM_TMPFOLDER}/status-check.${NTYPE}.done - -} diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 57ac34da..676f5f64 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -864,110 +864,34 @@ GUI(){ dl) DOWNLIST ${2} ;; - statusnew) - if [ "${2}" == "freespace" ]; then - STATUSNEW freespace ${3} - else - if [ -f "${NM_TMPFOLDER}/status-check.running" ] && [ $(expr `date +%s` - $(stat -c %Y ${NM_TMPFOLDER}/status-check.running)) -gt 1800 ]; then - # if [ -f "${NM_TMPFOLDER}/status-check.running" ]; then - SENDNOTICE "Nodemgmt Script Error" "Status monitoring stuck, system resetting... fix yo shit" - mv -f ${NM_TMPFOLDER}/status-check.running ${NM_TMPFOLDER}/status-check.running.crash - getStuckNM=$(pgrep -f '/nodemgmt') - for i in $getStuckNM; do - [ "$BASHPID" != "$i" ] && /bin/kill -9 $i - done - fi - - if [ ! -f "${NM_TMPFOLDER}/status-check.running" ]; then - touch "${NM_TMPFOLDER}/status-check.running" - if [ "${2}" == "report" ]; then - # start_status=`date +%s` - echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE} - echo -e "$(date +%Y-%m-%d-%H-%M-%S) - Starting status check #############################\n" > ${NM_TMPFOLDER}/status-check.running - STATUSNEW report ${3} >> ${NM_TMPFOLDER}/status-check.running - echo -e "\n\n$(date) - Finished status check #############################\n" >> ${NM_TMPFOLDER}/status-check.running - echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE} - - rm -f ${NM_LOGFOLDER}/status-check.new.lastrun - mv ${NM_TMPFOLDER}/status-check.running ${NM_LOGFOLDER}/status-check.new.lastrun - - # end_status=`date +%s` - # runtime_status=$((end_status-start_status)) - # echo -e "\nRuntime: ${runtime_status}" >> ${NM_LOGFOLDER}/status-check.lastrun - - if [ "${3}" == "email" ]; then - echo -e "\n\n" >> ${NM_LOGFOLDER}/status-check.new.lastrun - DOWNLIST report >> ${NM_LOGFOLDER}/status-check.new.lastrun - STATUSLOG="$(cat ${NM_LOGFOLDER}/status-check.new.lastrun)" - SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}" - fi - else - STATUSNEW ${2} ${3} - rm -f ${NM_TMPFOLDER}/status-check.running - fi - else - # echo -e "${idsCL[LightYellow]}Status check already running, please wait and try again${idsCL[Default]}" - echo -e "${idsCL[LightYellow]}Status checks are already running${idsCL[Default]}" - echo -en "${idsCL[LightCyan]}View the (L)ive log, or the (P)revious run log instead? ${idsCL[Default]}" - read -n1 choice - case $choice in - [Ll]) watch -n1 tail -n35 ${NM_TMPFOLDER}/status-check.running; echo;; - [Pp]) echo; cat ${NM_LOGFOLDER}/status-check.new.lastrun;; - esac - echo - fi - fi;; status) if [ "${2}" == "freespace" ]; then STATUS freespace ${3} else - if [ -f "${NM_TMPFOLDER}/status-check.running" ] && [ $(expr `date +%s` - $(stat -c %Y ${NM_TMPFOLDER}/status-check.running)) -gt 1800 ]; then - # if [ -f "${NM_TMPFOLDER}/status-check.running" ]; then - SENDNOTICE "Nodemgmt Script Error" "Status monitoring stuck, system resetting... fix yo shit" - mv -f ${NM_TMPFOLDER}/status-check.running ${NM_TMPFOLDER}/status-check.running.crash - getStuckNM=$(pgrep -f '/nodemgmt') - for i in $getStuckNM; do - [ "$BASHPID" != "$i" ] && /bin/kill -9 $i - done - fi - - if [ ! -f "${NM_TMPFOLDER}/status-check.running" ]; then - touch "${NM_TMPFOLDER}/status-check.running" - if [ "${2}" == "report" ]; then - # start_status=`date +%s` - echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE} - echo -e "$(date +%Y-%m-%d-%H-%M-%S) - Starting status check #############################\n" > ${NM_TMPFOLDER}/status-check.running - STATUS report ${3} >> ${NM_TMPFOLDER}/status-check.running - echo -e "\n\n$(date) - Finished status check #############################\n" >> ${NM_TMPFOLDER}/status-check.running - echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE} - - rm -f ${NM_LOGFOLDER}/status-check.lastrun - mv ${NM_TMPFOLDER}/status-check.running ${NM_LOGFOLDER}/status-check.lastrun - - # end_status=`date +%s` - # runtime_status=$((end_status-start_status)) - # echo -e "\nRuntime: ${runtime_status}" >> ${NM_LOGFOLDER}/status-check.lastrun - - if [ "${3}" == "email" ]; then - echo -e "\n\n" >> ${NM_LOGFOLDER}/status-check.lastrun - DOWNLIST report >> ${NM_LOGFOLDER}/status-check.lastrun - STATUSLOG="$(cat ${NM_LOGFOLDER}/status-check.lastrun)" - SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}" - fi - else - STATUS ${2} ${3} - rm -f ${NM_TMPFOLDER}/status-check.running + status_start=$(date +%Y-%m-%d-%H-%M-%S) + STATUSRUN_TMPFOLDER=${NM_TMPFOLDER}/${status_start} + mkdir ${STATUSRUN_TMPFOLDER} + touch "${STATUSRUN_TMPFOLDER}/status-check.running" + if [ "${2}" == "report" ]; then + echo "${status_start} - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE} + echo -e "${status_start} - Starting status check #############################\n" > ${STATUSRUN_TMPFOLDER}/status-check.running + STATUS report ${3} >> ${STATUSRUN_TMPFOLDER}/status-check.running + echo -e "\n\n$(date +%Y-%m-%d-%H-%M-%S) - Finished status check #############################\n" >> ${STATUSRUN_TMPFOLDER}/status-check.running + echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE} + + rm -f ${NM_LOGFOLDER}/status-check.lastrun + mv ${STATUSRUN_TMPFOLDER}/status-check.running ${NM_LOGFOLDER}/status-check.lastrun + rm -Rf ${STATUSRUN_TMPFOLDER} + + if [ "${3}" == "email" ]; then + echo -e "\n\n" >> ${NM_LOGFOLDER}/status-check.lastrun + DOWNLIST report >> ${NM_LOGFOLDER}/status-check.lastrun + STATUSLOG="$(cat ${NM_LOGFOLDER}/status-check.lastrun)" + SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}" fi else - # echo -e "${idsCL[LightYellow]}Status check already running, please wait and try again${idsCL[Default]}" - echo -e "${idsCL[LightYellow]}Status checks are already running${idsCL[Default]}" - echo -en "${idsCL[LightCyan]}View the (L)ive log, or the (P)revious run log instead? ${idsCL[Default]}" - read -n1 choice - case $choice in - [Ll]) watch -n1 tail -n35 ${NM_TMPFOLDER}/status-check.running; echo;; - [Pp]) echo; cat ${NM_LOGFOLDER}/status-check.lastrun;; - esac - echo + STATUS ${2} ${3} + rm -f ${NM_TMPFOLDER}/status-check.running fi fi;; scantimes) STATUS_SCANTIMES ${2} ${3} ${4} ${5};;