voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity
Files
master
NodeMgmt/inc/status.inc
David Schroeder b798e39c72 update
2025-10-01 22:07:04 -05:00

1246 lines
66 KiB
Bash
Executable File
Raw Permalink Blame History

#!/usr/bin/env bash
STATUS(){
start=$(date +%s)
log_start=$(date "+%Y-%m-%d %H:%M:%S")
[ "${NM_SCAN_THREADS}" != "" ] && SCAN_THREADS=${NM_SCAN_THREADS} || SCAN_THREADS=4
if [ "${1}" != "" ] && [ "${NM_NODETYPES[${1^^}]}" != "" ]; then
ntss=${1^^}
ntypesel=(${ntss})
STATUS_ACTION=${2}
else
ntss=""
ntypesel=(${NM_NODE_TYPES[@]})
[ "${1}" == "sync" ] && STATUS_ACTION=repl || STATUS_ACTION=${1}
fi
# while [ $# -gt 0 ]; do
# case "$1" in
# -site) DEL_SITE=${2};;
# -ssl) DEL_SSL=${2};;
#
# esac
# shift
# done
PRI_CW=40
LD_CW=62
DV_LEN=80
SV_LEN=55
declare -i cw; declare -i spc1; declare -i c
############################
## REPLICATION CHECK SETUP
############################
# REPLRUN=0
([ "${STATUS_ACTION}" != "services" ] && [ "${NM_DISABLE_REPL_CHECK}" != "true" ] && ([ ! -f ${NM_TMPFOLDER}/.replcheck ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replcheck)) -ge $(expr ${NM_REPL_CHECK_TIMEOUT} \* 60) ])) && REPLRUN=1 || REPLRUN=0
([ -f ${NM_TMPFOLDER}/.replcheck.fail ] && [ ${REPLRUN} -eq 0 ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replcheck.fail)) -ge 60 ]) && REPLRUN=1
[ ${REPLRUN} -eq 1 ] && touch ${NM_TMPFOLDER}/.replcheck
# if [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "" ]; then
if [ "${STATUS_ACTION}" == "repl" ] || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && [ "${NM_REPL_CHECK[${ntss}]}" != "" ] && [ "${STATUS_ACTION}" == "repl" ]); then
touch ${NM_TMPFOLDER}/.replcheck
if [ "${STATUS_ACTION}" != "check" ]; then
MSG="Setting up replication checks"
c=0; cw=${LD_CW}; spc=''; spc1=${cw}-${#MSG}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${MSG}${spc}: "
fi
for NTYPE in "${ntypesel[@]}"; do
REPLSTART ${NTYPE} &
done
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"
MSG="Starting processes to collect/monitor replication checks"
c=0; cw=${LD_CW}; spc=''; spc1=${cw}-${#MSG}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${MSG}${spc}: "
fi
for NTYPE in "${ntypesel[@]}"; do
PH=${NTYPE}_HOSTS[0]
PHA=${NTYPE}_HOSTS
skip=0
if [ -f ${NM_TMPFOLDER}/${!PH}.disable ] || [ -f ${NM_TMPFOLDER}/${!PH}.skip ] || [ -f ${NM_TMPFOLDER}/${!PH}.down ]; then
if [ $(eval echo \${#${PHA}[@]}) -eq 2 ]; then
skip=1
else
PH=${NODETYPE}_HOSTS[1]
fi
fi
if [ ${skip} -eq 0 ]; then
PH_CMD="${SSHCMD} root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ] && [ ! -f ${NM_TMPFOLDER}/${!PH}.disable ] && [ ! -f ${NM_TMPFOLDER}/${!PH}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
until [ $(ls ${STATUSRUN_TMPFOLDER}/repl.*.running 2>/dev/null | wc -l) -lt ${SCAN_THREADS} ]; do tmp=tmp; done
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${STATUS_ACTION}" & >/dev/null 2>&1
done
fi
fi
done
fi
fi
done
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"
fi
if [ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "services" ] || [ "${STATUS_ACTION}" == "dockers" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "all" ]; then
############################
## SERVICE/DOCKER CHECK SETUP
############################
MSG="Starting processes to collect node information"
c=0; cw=${LD_CW}; spc=''; spc1=${cw}-${#MSG}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]}${MSG}${spc}: "
for NTYPE in "${ntypesel[@]}"; do
until [ $(ls ${STATUSRUN_TMPFOLDER}/status-check.*.running 2>/dev/null | wc -l) -lt ${SCAN_THREADS} ]; do tmp=tmp; done
STATUS_NODE ${NTYPE} > ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.results 2>/dev/null &
done
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"
########################
## LOCALHOST CHECK
########################
if [ ! -z ${LOCAL_SERVICES+x} ] && [ "${STATUS_ACTION}" != "dockers" ] && [ "${ntss}" == "" ]; then
cpu_usage=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
if [ "${STATUS_ACTION}" != "check" ]; then
lip=${RUN_NODE_IP}
echo -e "\n$(DIVIDER . . ${DV_LEN})\n${idsCL[Yellow]}${idsST[Bold]} LOCALHOST Service Status${idsST[Reset]}${idsCL[Default]}"
echo -e "$(DIVIDER . . ${DV_LEN})"
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} ${idsST[Reset]}${idsCL[LightCyan]}[${lip}]"
if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then
uptime=`uptime -p`
echo -e "${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
else
echo -e "${idsCL[Default]}"
fi
DIVIDER . green `expr ${DV_LEN} - 10`
fi
# if [ "${STATUS_ACTION}" != "check" ]; then
# lip=${RUN_NODE_IP}
# uptime=`uptime -p`
# echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
# DIVIDER false green ${DV_LEN}
# fi
for srvc in "${LOCAL_SERVICES[@]}"; do
if [ "${STATUS_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "report" ]; then
c=0; cw=${PRI_CW}; spc=''
spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
fi
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down))
if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
[ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
fi
fi
else
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}"
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
else
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down))
fi
rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE}
fi
fi
done
fi
###################################
## NODE SERVICE AND DOCKER CHECK
###################################
nc_count=0; completed=false; scanstart=$(date +%s)
until [ "${completed}" == "true" ]; do
for nodestatus in ${STATUSRUN_TMPFOLDER}/status-check.*.done; do
NTS=$(grep -oP '(?<=status-check.).*?(?=.done)' <<< "${nodestatus}")
if [ "${NTS}" != "*" ]; then
nodestatus=$(cat ${STATUSRUN_TMPFOLDER}/status-check.${NTS}.results)
[ "${nodestatus}" != "" ] && echo -e "${nodestatus}"
rm -f ${STATUSRUN_TMPFOLDER}/status-check.${NTS}.done
((nc_count++))
# echo "${NTS} == ${nc_count} == ${nodestatus}"
fi
done
if [ $(($(date +%s)-scanstart)) -gt 600 ]; then
SENDNOTICE "NMG Service/Docker Scan Stuck" "NMG Service/Docker scan has been running for 10mins, klling all bash...." 1
killall bash
exit 1
fi
if [ ${nc_count} -eq ${#ntypesel[@]} ]; then
completed=true
break
fi
sleep 1s
done
# for NTYPE in "${ntypesel[@]}"; do
# completed=false
# until [ "${completed}" == "true" ]; do
# if [ -f ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.done ]; then
# completed=true
# echo -e "$(cat ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.results)"
# fi
# done
# done
fi
########################
## REPLICATION CHECK
########################
# if [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "" ]; then
# if [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "" ] || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]); then
if [ "${STATUS_ACTION}" == "repl" ] || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && [ "${NM_REPL_CHECK[${ntss}]}" != "" ] && [ "${STATUS_ACTION}" == "repl" ]); then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER . . ${DV_LEN}
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER . . ${DV_LEN}; echo -en "${idsST[Reset]}"
fi
for NTYPE in "${ntypesel[@]}"; do
PH=${NTYPE}_HOSTS[0]
PHA=${NTYPE}_HOSTS
skip=0
if [ -f ${NM_TMPFOLDER}/${!PH}.disable ] || [ -f ${NM_TMPFOLDER}/${!PH}.down ] || [ -f ${STATUSRUN_TMPFOLDER}/repl.${!PH}.skip ]; then
if [ $(eval echo \${#${PHA}[@]}) -eq 2 ]; then
skip=1
else
PH=${NODETYPE}_HOSTS[1]
fi
fi
PH_CMD="${SSHCMD} root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ] && [ ! -f ${STATUSRUN_TMPFOLDER}/repl.${!PH}.skip ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]}[${!PH}] <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green `expr ${DV_LEN} - 10`
fi
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
if [ "${STATUS_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "report" ]; then
c=0; cw=${PRI_CW}; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${idsCL[White]}${NM_REPL_DESC[${rcheck}]}${spc}${idsCL[Default]}: "
fi
checked=""; scanstart=$(date +%s)
until [ "${checked}" != "" ]; do
if [ -f ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
checked=good
elif [ -f ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
checked=timeout
elif [ ${skip} -eq 1 ] || [ -f ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.skip ] || [ -f ${STATUSRUN_TMPFOLDER}/repl.${nip}.skip ] || [ -f ${STATUSRUN_TMPFOLDER}/repl.${!PH}.skip ]; then
checked=skip
fi
if [ $(($(date +%s)-scanstart)) -gt 600 ]; then
SENDNOTICE "NMG Scan Stuck" "NMG Status Replication Check has been running for 10mins, klling all bash...." 1
touch ${NM_TMPFOLDER}/.replcheck.fail
mv ${STATUSRUN_TMPFOLDER} ${STATUSRUN_TMPFOLDER}.stuck
killall bash
exit 1
fi
done
if [ "${checked}" == "timeout" ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
else
# if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] && ([[ "${nip}" != *"10.2."* ]] || [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ]) && ([ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)) -gt 60 ]); then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] && ([ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)) -gt 60 ]); then
[ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ]; then
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
It has been down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)))" 1
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent
fi
fi
fi
elif [ "${checked}" == "good" ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}"
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ]; then
SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)))"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}
fi
rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl*
fi
${SSHCMD} root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' &
elif [ "${checked}" == "skip" ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightCyan]}Skipped${idsCL[Default]}"
fi
done
else
if [ "${STATUS_ACTION}" != "check" ]; then
if [ -f ${STATUSRUN_TMPFOLDER}/repl.${!PH}.skip ]; then
echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is skipped${idsCL[Default]}"
elif [ -f ${NM_FOLDER}/${!PH}.down ]; then
echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
elif [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
fi
fi
fi
[ "${STATUS_ACTION}" != "check" ] && echo
fi
done
fi
done
##########################
# REMOVE REPL CHECK FILES
##########################
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}"
for NTYPE in "${ntypesel[@]}"; do
PH=${NTYPE}_HOSTS[0]
PHA=${NTYPE}_HOSTS
skip=0
if [ -f ${NM_TMPFOLDER}/${!PH}.disable ] || [ -f ${NM_TMPFOLDER}/${!PH}.down ]; then
if [ $(eval echo \${#${PHA}[@]}) -eq 2 ]; then
skip=1
else
PH=${NODETYPE}_HOSTS[1]
fi
fi
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ${skip} -eq 0 ]; then
PH_CMD="${SSHCMD} root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
if [ "${PH_CMD}" == "" ]; then
rm -f ${NM_FOLDER}/.test.${STATUS_START//-/}.repl &
for rcheck in "${!var}"; do
rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl &
done
else
${PH_CMD} rm -f ${NM_FOLDER}/.test.${STATUS_START//-/}.repl &
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl &
done
fi
fi
fi
done
if [ ! -f ${NM_TMPFOLDER}/.replclean ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replclean)) -ge 10800 ]; then
for NTYPE in "${NM_NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
PHA=${NTYPE}_HOSTS
skip=0
if [ -f ${NM_TMPFOLDER}/${!PH}.disable ] || [ -f ${NM_TMPFOLDER}/${!PH}.down ]; then
if [ $(eval echo \${#${PHA}[@]}) -eq 2 ]; then
skip=1
else
PH=${NODETYPE}_HOSTS[1]
fi
fi
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ${skip} -eq 0 ]; then
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
PH_CMD="${SSHCMD} root@${!PH}"
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.2023*.repl
if [ $(${PH_CMD} ls ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.*.repl 2>/dev/null | wc -l) -gt 0 ]; then
for replfile in $(${PH_CMD} ls ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.*.repl); do
[ $(expr $(date +%s) - $(${PH_CMD} stat -c %Y ${replfile})) -ge 1200 ] && ${PH_CMD} rm -f ${replfile} &
done
fi
done
fi
fi
done
REPLRUN=2
touch ${NM_TMPFOLDER}/.replclean
fi
rm -f ${NM_FOLDER}/.test.${STATUS_START//-/}.repl
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}\n"
touch ${NM_TMPFOLDER}/.replcheck
rm -f ${NM_TMPFOLDER}/.replcheck.fail
fi
########################
## FREE SPACE CHECK
########################
#if [ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "freespace" ]; then
if [ "${STATUS_ACTION}" == "freespace" ]; then
for NTYPE in "${ntypesel[@]}"; do
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsST[Bold]}"; DIVIDER . . ${DV_LEN}
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node(s) Free Space Scan${idsCL[Default]}"
DIVIDER . . ${DV_LEN}; echo -e "${idsST[Reset]}"
fi
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="${SSHCMD} root@${nip}"; LH=''
fi
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
DIVIDER false green `expr ${DV_LEN} - 10`
fi
[ "${STATUS_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
declare -A partitions
DRIVEINFO=$(${SSHCMD} root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
DRIVEINFO=(${DRIVEINFO})
if [ "${2}" != "report" ]; then
echo -en "\e[1A"
echo -e "\e[0K\r"
fi
NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
ii=$((${i}*3))
pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
pname=${pname#*vg-}
if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then
freespace=${DRIVEINFO[$((${ii}+2))]//M/}
if [ "${STATUS_ACTION}" != "check" ]; then
c=0; cw=20; spc=''
spc1=${cw}-${#pname}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${pname}$spc: "
fi
if [[ ${freespace} -gt 1024 ]]; then
fsgb=$(bc <<< "scale=2; ${freespace}/1024")
fsdsp="${fsgb} GB"
else
fsdsp="${freespace} MB"
fi
if [ "${freespace}" -le "1024" ]; then
fs_status='error'
fs_status_color='Red'
SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free" 1
elif [ "${freespace}" -le "5120" ]; then
fs_status='warn'
fs_status_color='Yellow'
SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free"
else
fs_status=''
fs_status_color='Green'
fi
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
fi
done
echo
done
done
fi
########################
if [ "${STATUS_ACTION}" != "check" ]; then
echo
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" == "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
fi
end=`date +%s`; runtime=$((end-start)); echo -e "Runtime: ${runtime}\n"
[ "${STATUS_ACTION}" == "report" ] && echo "${log_start}~${runtime}~${REPLRUN}" >> ${NM_LOGFOLDER}/status-check.scantimes
}
STATUS_NODE(){
NODETYPE=${1}
dockers=${NODETYPE}_DOCKERS_CHECK[@]
hosts=${NODETYPE}_HOSTS[@]
NTHOSTS=${NODETYPE}_HOSTS[@]
touch ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.running
if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${STATUS_ACTION}" == "services" ]); then
GOFORCHECK=false;
else
GOFORCHECK=true;
fi
if [ ${GOFORCHECK} = true ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER . . ${DV_LEN}; DIVIDER . . ${DV_LEN}
if [ "${STATUS_ACTION}" == "dockers" ]; then
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
else
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
fi
DIVIDER . lightYellow ${DV_LEN}; echo -en "${idsST[Reset]}"
fi
for nip in "${!NTHOSTS}"; do
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}"
[ "${STATUS_ACTION}" != "report" ] && echo -en " - Verifying Host..."
fi
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
NCMD=''; LH='- localhost'
else
NCMD="${SSHCMD} root@${nip}"; LH=''
fi
if [ "${NCMD}" != "" ] && [ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.* ]; then
checkhost=$(CHECK_HOST ${nip})
[ "${checkhost}" != "false" ] && checkhostssh=$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1)
elif [ -f ${STATUSRUN_TMPFOLDER}/.${nip}.up ]; then
checkhost=true
checkhostssh=ok
elif [ -f ${STATUSRUN_TMPFOLDER}/.${nip}.down ]; then
checkhost=false
checkhostssh=no
else
checkhost=true
checkhostssh=ok
fi
if [ "${checkhost}" != "false" ] && [ "${checkhostssh}" == "ok" ]; then
if [ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.up ]; then
############## ALERTING ON HIGH CPU USAGE ##############
# [ "${NODETYPE}" != "OFW" ] && cpu_usage=$(${NCMD} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage=""
# if [ "${cpu_usage}" != "" ]; then
# if [ "${NM_HOST_CPULEVELS[${nip}]}" != "" ]; then
# cpu_warn=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 1)
# cpu_crit=$(echo ${NM_HOST_CPULEVELS[${nip}]} | cut -d',' -f 2)
# else
# cpu_warn=75
# cpu_crit=85
# fi
# if [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_warn} ]; then
# CUFC="${idsCL[Green]}"
# if [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then
# if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ]; then
# echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.norm
# elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]); then
# [ -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] NORMAL CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has returned to normal cpu usage: ${cpu_usage}%"
# rm -f ${NM_TMPFOLDER}/${nip}.cpu_usage.*
# fi
# fi
# elif [ $(ROUND_NUMBER ${cpu_usage}) -lt ${cpu_crit} ]; then
# CUFC="${idsCL[LightYellow]}"
# if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ]; then
# echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn
# elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] WARNING HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has high cpu usage: ${cpu_usage}%"
# echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.warn
# touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent
# fi
# else
# CUFC="${idsCL[LightRed]}"
# if [ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.crit ]; then
# echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit
# elif [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.crit)) -ge 300 ] && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.warn ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.warn)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.norm ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.norm)) -ge 300 ]) && ([ ! -f ${NM_TMPFOLDER}/${nip}.cpu_usage.sent ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.cpu_usage.sent)) -ge 3600 ]); then
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}] CRITICAL HIGH CPU USAGE" "${NM_HOSTNAMES[${nip}]}[${nip}] has CRITICALLY high cpu usage: ${cpu_usage}%" 1
# echo "${cpu_usage}%" >| ${NM_TMPFOLDER}/${nip}.cpu_usage.crit
# touch ${NM_TMPFOLDER}/${nip}.cpu_usage.sent
# fi
# fi
# fi
touch ${STATUSRUN_TMPFOLDER}/.${nip}.up
fi
if [ "${STATUS_ACTION}" != "check" ]; then
if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then
if [ "${NODETYPE}" == "OFW" ]; then
uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1)
if [ ${#uptime} -ge 6 ]; then
uptime="up $(echo $uptime | cut -d: -f1) days, $(echo $uptime | cut -d: -f2) hours, $(echo $uptime | cut -d: -f3) minutes"
elif [ ${#uptime} -ge 3 ]; then
uptime="up $(echo $uptime | cut -d: -f1) hours, $(echo $uptime | cut -d: -f2) minutes"
else
uptime="up ${uptime} minutes"
fi
else
uptime=$(${NCMD} uptime -p)
fi
[ "${STATUS_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${CUFC}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
else
echo -e "${idsCL[Default]}"
fi
DIVIDER false lightCyan ${DV_LEN}
fi
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.sent ]; then
[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
rm -f ${NM_TMPFOLDER}/${nip}.*
fi
########################
## SERVICES CHECK
########################
if [ "${STATUS_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then
echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}"
DIVIDER false green ${SV_LEN}
srvcs=${NODETYPE}_SERVICES_CHECK[@];
srvcstotest="$(join_by " " ${!srvcs})"
if [ "${NODETYPE}" != "OFW" ]; then
[ "${NCMD}" != "" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}" | tr -d $'\r'`) || srvctst=(`systemctl is-active ${srvcstotest}`)
fi
sr=0
[ "${NODETYPE}" == "MM" ] && mmstop=$(${SSHCMD} root@${nip} "if [ -f /opt/idssys/mediamanager/.tmp/mm.stop ] || [ -f /opt/idssys/mediamanager/.tmp/mm.noservices ]; then echo 1; fi")
for srvc in "${!srvcs}"; do
if [ "${STATUS_ACTION}" != "report" ]; then
c=0; cw=${PRI_CW}; spc=''
spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
tmp=${NM_SINGLESRVR_SERVICES[${NODETYPE}]}
SSCHK=true
if [ "$(GETSKIP ${nip} host)" == "true" ]; then
SSCHK=skip
elif [[ ${tmp[@]} =~ ${srvc} ]]; then
if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then
SSCHK=false
else
SSCHK=primary
fi
elif ([ "${nip}" == "10.2.1.51" ] && [ "${srvc}" == "keepalived" ]); then
SSCHK=disable
fi
if [ "${SSCHK}" == "true" ] && [ "${NODETYPE}" == "MM" ]; then
[ ${mmstop} -eq 1 ] && [[ "${NM_MEDIA_SERVICES}" = *"${srvc}"* ]] && SSCHK=skip
fi
([ "${SSCHK}" == "skip" ] || [ "${SSCHK}" == "false" ] || [ "${SSCHK}" == "disable" ]) && rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && ([ ${SSCHK} == true ] || [ ${SSCHK} == primary ]); then
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
if [ "${NODETYPE}" != "OFW" ]; then
srvctest=${srvctst[$sr]}
if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then
[ "$(${NCMD} systemctl is-active mariadb | tr -d $'\r')" == "active" ] && mysqlgo=true || mysqlgo=false
elif [ "${srvctest}" != "active" ] && [ "${srvc}" == "mariadb" ]; then
[ "$(${NCMD} systemctl is-active mysql | tr -d $'\r')" == "active" ] && mysqlgo=true || mysqlgo=false
elif [ "${srvctest}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then
mysqlgo=true
fi
else
if [ "$(${SSHCMD} root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
srvctest=active
else
srvctest=notactive
fi
fi
if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running"
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
# [ "${srvc}" == "pdns" ] && ${NCMD} systemctl restart ${srvc} &
else
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})"
#if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is down" 1
touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent
fi
fi
#########################################
############ SERVICE REPAIRS ############
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 90 ]; then
if [ "${srvc}" == "pihole-FTL" ]; then
${SSHCMD} root@${nip} 'pihole -r' & >/dev/null 2>&1
touch ${NM_TMPFOLDER}/${nip}~${srvc}.fix
fi
fi
#########################################
#########################################
fi
else
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.fix ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - was FIXED! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} was FIXED, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
else
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
fi
fi
fi
if [[ "${NM_SINGLESRVR_SERVICES[${NODETYPE}]}" = *"${srvc}"* ]]; then
for tnip in "${!NTHOSTS}"; do
rm -f ${NM_TMPFOLDER}/${tnip}~${srvc}.* &
done
else
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* &
fi
fi
[ "${mysqlgo}" == "true" ] && unset mysqlgo
[ "${SSCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}"
# elif [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] || ([ "${nip}" != "10.2.1.5" ] && [[ "${nip}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ]) || ([ "${srvc}" == "keepalived" ] && [ "${nip}" == "10.2.1.51" ]); then
elif [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] || [ "${SSCHK}" == "disable" ]; then
echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled${idsCL[Default]}"
elif [ "${SSCHK}" == "skip" ]; then
echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Host Disabled${idsCL[Default]}"
else
echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Backup Node${idsCL[Default]}"
fi
sr=`expr $sr + 1`
done
fi
########################
## DOCKER CHECK
########################
if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" != "services" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
[ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo
echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}"
DIVIDER false green ${SV_LEN}
fi
for docker in "${!dockers}"; do
if [ "${STATUS_ACTION}" != "report" ]; then
c=0; cw=${PRI_CW}; spc=''
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
tmp=${NM_SINGLESRVR_DOCKERS[${NODETYPE}]}
SDCHK=true
if ${SSHCMD} root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
SDCHK=skip
elif [[ ${tmp[@]} =~ ${docker} ]]; then
if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then
SDCHK=false
else
SDCHK=primary
fi
elif [ "${nip}" != "10.10.10.240" ] && [[ "${docker}" = "hcloud_"* ]]; then
SDCHK=disable
elif [ "${nip}" == "10.10.1.180" ] && ([ "${docker}" == "watchtower" ] || [ "${docker}" == "portainer_agent" ]); then
SDCHK=disable
fi
([ "${SDCHK}" == "skip" ] || [ "${SDCHK}" == "false" ]) && rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && ([ ${SDCHK} == true ] || [ ${SDCHK} == primary ]); then
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
# docker_go=0
# if [ "$(${NCMD} docker inspect -f {{.State.Health.Status}} ${docker} 2>/dev/null)" == "healthy" ]; then
# docker_go=1
# elif [ "$(${NCMD} docker inspect -f {{.State.Running}} ${docker})" == "true" ]; then
# docker_go=1
# fi
if [ "${NCMD}" == "" ]; then
if [ "$(docker inspect -f {{.State.Health.Status}} ${docker} 2>/dev/null)" == "healthy" ] || [ "$(docker inspect -f {{.State.Running}} ${docker} 2>/dev/null)" == "true" ]; then
docker_go=true
else
docker_go=false
fi
else
docker_go=$(${NCMD} "docker_name='${docker}'; if [ \"\$(docker inspect -f {{.State.Health.Status}} \${docker_name} 2>/dev/null)\" == \"healthy\" ] || [ \"\$(docker inspect -f {{.State.Running}} \${docker_name} 2>/dev/null)\" == \"true\" ]; then echo true; else echo false; fi")
fi
if [ "${docker_go}" == "false" ]; then
#if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
if [ "${NM_DOCKER_COMPOSE_LOC[${docker}]}" != "" ]; then
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, will try and fix in 1min" >> ${NM_LOGFILE}
echo -en "${idsCL[Red]}Not Running - will try and fix in 1min!"
else
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down" >> ${NM_LOGFILE}
echo -en "${idsCL[Red]}Not Running"
fi
elif ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)) -gt 50 ]) || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.fix)) -ge 3600 ]); then
[ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
elif [ "${NM_DOCKER_COMPOSE_LOC[${docker}]}" != "" ] || [ "${docker}" == "watchtower" ] || [ "${docker}" == "portainer_agent" ]; then
if [ "${docker}" == "watchtower" ]; then
${NCMD} runup watchtower >/dev/null 2>&1
elif [ "${docker}" == "portainer_agent" ]; then
${NCMD} docker run -d -p 9001:9001 --name portainer_agent --restart=always -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker/volumes:/var/lib/docker/volumes -v /:/host portainer/agent:latest >/dev/null 2>&1
else
${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
fi
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
touch ${NM_TMPFOLDER}/${nip}~${docker}.fix
sleep 10s
if [ "$(${NCMD} docker ps -a -q -f name=${docker})" ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Green]}Running - Fixed"
[ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed"
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
else
if [ "${STATUS_ACTION}" != "check" ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is down, could not fix" 1
touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
fi
fi
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then
# elif [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && ([ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] || ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ])); then
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_SERVICE_DESC[${docker}]} - is down" 1
touch ${NM_TMPFOLDER}/${nip}~${docker}.sent
fi
echo -en "${idsCL[Red]}Not Running, could not fix earlier, FIX ME!"
fi
else
echo -en "${idsCL[Yellow]}Disabled - Skipping"
fi
else
if [ -f ${NM_TMPFOLDER}/${nip}~${docker}.fix ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${docker}.sent ] && SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed"
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Green]}Running - Fixed"
else
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
fi
if [[ "${NM_SINGLESRVR_DOCKERS[${NODETYPE}]}" = *"${docker}"* ]]; then
for tnip in "${!NTHOSTS}"; do
rm -f ${NM_TMPFOLDER}/${tnip}~${docker}.* &
done
else
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* &
fi
fi
[ "${SDCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}"
elif [ -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] || [ "${SDCHK}" == "disable" ]; then
echo -e "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled${idsCL[Default]}"
elif [ "${SDCHK}" == "skip" ]; then
echo -e "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Host Disabled${idsCL[Default]}"
else
echo -e "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Backup Node${idsCL[Default]}"
fi
done
fi
else
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightRed]} - Offline${idsCL[Default]}"
DIVIDER false lightCyan ${DV_LEN}
fi
[ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.down ] && touch ${STATUSRUN_TMPFOLDER}/.${nip}.down
rm -f ${NM_TMPFOLDER}/${nip}~*
([[ "${NM_IGNOREHOSTS_IFDOWN}" = *"${nip}"* ]] && [ ! -f ${NM_TMPFOLDER}/${nip}.disable ]) && touch ${NM_TMPFOLDER}/${nip}.disable && touch ${NM_TMPFOLDER}/${nip}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
# if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then
[ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && touch ${NM_TMPFOLDER}/${nip}.down
if [ "${STATUS_ACTION}" != "check" ]; then
if [ "${checkhost}" != "false" ]; then
echo -e "${idsCL[Red]} Node is online, but SSH is down!${idsCL[Default]}"
else
echo -e "${idsCL[Red]} Node is down! ${idsCL[Default]}"
fi
fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
[ "${checkhostssh}" != "ok" ] && echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
else
#if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) -gt 60 ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) -gt 60 ]; then
mv ${NM_TMPFOLDER}/${nip}.down ${NM_TMPFOLDER}/${nip}.errtime
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
errtime=${toterrtime}
if [ "${checkhost}" != "false" ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is online, but SSH is down" 1
else
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
fi
touch ${NM_TMPFOLDER}/${nip}.sent
else
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=${errtime}
fi
if [ "${STATUS_ACTION}" != "check" ]; then
if [ "${checkhost}" != "false" ]; then
echo -e "${idsCL[LightRed]} Node SSH has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
else
echo -e "${idsCL[LightRed]} Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
fi
fi
# if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then
if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then
if [ "${checkhost}" != "false" ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSH has been down for $(SHOW_TIME ${toterrtime})" 1
else
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
fi
touch ${NM_TMPFOLDER}/${nip}.sent
fi
#if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then
[ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && touch ${NM_TMPFOLDER}/${nip}.down
fi
fi
[ "${STATUS_ACTION}" != "check" ] && echo
done
fi #GOFORCHECK
mv ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.running ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.done
}
REPLCHECK(){
rcheck=${1}
nip=${2}
PH_CMD=${3}
touch ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.running
# if [ "${4}" != "check" ]; then
# timeout=`date --date='1 minutes' +%s`
# # timeout=`date --date='30 seconds' +%s`
# else
# timeout=`date --date='2 minutes' +%s`
# fi
timeout=`date --date='1 minutes' +%s`
# timeout=`date --date='30 seconds' +%s`
# if [ -f ${NM_TMPFOLDER}/${nip}.disable ] || [ -f ${NM_TMPFOLDER}/${nip}.down ] || ([[ "${nip}" = "10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ]) ||
# if [ -f ${NM_TMPFOLDER}/${nip}.disable ] || [ -f ${NM_TMPFOLDER}/${nip}.down ] || ([ "${nip}" == "10.10.10.240" ] && [ "${rcheck}" == "rootssh" ]); then
if [ -f ${NM_TMPFOLDER}/${nip}.disable ] || [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
rm -f ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.running
touch ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.skip
else
checked=false
until [ "${checked}" == "" ]; do
if [ "${PH_CMD}" == "" ]; then
${SSHCMD} root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked=`${SSHCMD} root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl`
else
${SSHCMD} root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked="`${PH_CMD} \"${SSHCMD} root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\"`"
fi
if [ "`date +%s`" -gt "${timeout}" ]; then
timeout=true
break
fi
done
rm -f ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.running
[ "${timeout}" == "true" ] && touch ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${STATUSRUN_TMPFOLDER}/repl.${rcheck}.${nip}.good
fi
}
REPLSTART(){
NODETYPE=${1}
PH=${NODETYPE}_HOSTS[0]
PHA=${NODETYPE}_HOSTS
skip=0
if [ -f ${NM_TMPFOLDER}/${!PH}.disable ] || [ -f ${NM_TMPFOLDER}/${!PH}.down ]; then
touch ${STATUSRUN_TMPFOLDER}/repl.${!PH}.skip
if [ $(eval echo \${#${PHA}[@]}) -eq 2 ]; then
skip=1
else
PH=${NODETYPE}_HOSTS[1]
if [ -f ${NM_TMPFOLDER}/${!PH}.disable ] || [ -f ${NM_TMPFOLDER}/${!PH}.down ]; then
skip=1
touch ${STATUSRUN_TMPFOLDER}/repl.${!PH}.skip
fi
fi
fi
if [ ${skip} -eq 0 ] && [ "${!PH}" != "" ]; then
PH_CMD="${SSHCMD} root@${!PH}"
var=${NODETYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${NM_FOLDER}/.test.${STATUS_START//-/}.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
${PH_CMD} "echo -e \"Replcation-Test\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl" &
${PH_CMD} 'find '${NM_REPL_CHECK_LOC[${rcheck}]}' -iname "*.sync-conflict-*" -exec rm {} \;' &
done
fi
fi
}
REMOVE_CONFLICT_SYNC_FILES(){
echo
if [ "${1}" != "" ] && [ "${NM_NODETYPES[${1^^}]}" != "" ]; then
ntss=${1^^}
ntypesel=(${ntss})
else
ntss=""
ntypesel=(${NM_NODE_TYPES[@]})
fi
for NTYPE in "${ntypesel[@]}"; do
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
echo -e "${idsCL[LightYellow]}Removing all Syncthing 'sync-conflict' files from ${idsCL[LightGreen]}${NM_HOSTNAMES[${nip}]}${idsCL[LightYellow]} [${nip}]${idsCL[Default]}"
PH_CMD="${SSHCMD} root@${nip}"
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
echo -en "Removing files from ${idsCL[LightCyan]}${NM_REPL_CHECK_LOC[${rcheck}]}/* ${idsCL[Default]}... "
${PH_CMD} 'find '${NM_REPL_CHECK_LOC[${rcheck}]}' -iname "*.sync-conflict-*" -exec rm {} \;' >/dev/null 2>&1
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
done
fi
echo
done
fi
done
echo
}
STATUS_SCANTIMES(){
start=$(date +%s)
scandefault=1440
while [ $# -gt 0 ]; do
case "${1}" in
-l|-last) LAST=${2};;
-h|-help|--help)
echo -e "Usage: ${idsCL[Yellow]}[nodemgmt or nmg] scantimes {opt}${idsCL[Default]} {"
width=15
printf "%-${width}s- %s\n" " -l|-last" "(number of last scan times to average from, defaults to ${scandefault})"
echo -e "}\n"
exit 0;;
esac
shift
done
[ "${LAST}" == "" ] && LAST=${scandefault}
echo -en "${idsCL[Yellow]}Pulling data and calculating average scan times for the last ${idsCL[LightYellow]}$(IDS_NUMBER_FORMAT ${LAST}) scans ${idsCL[Yellow]}... "
lastscantime=$(tail -n 1 ${NM_LOGFOLDER}/status-check.scantimes)
lastscantime=$(echo "${lastscantime}" | cut -d '~' -f 2)
scantimes=$(tail -n ${LAST} ${NM_LOGFOLDER}/status-check.scantimes)
longest_scantime=0
IFS=$'\n'
read -rd '' -a scantimes <<<"${scantimes}"
unset IFS
scantimes_norm=0; scansfound_norm=0
scantimes_norm_xtr=0; scansfound_norm_xtr=0
scantimes_repl=0; scansfound_repl=0
scantimes_repl_xtr=0; scansfound_repl_xtr=0
scantimes_replchk=0; scansfound_replchk=0
scantimes_replchk_xtr=0; scansfound_replchk_xtr=0
for scan in "${scantimes[@]}"; do
scantime=$(echo "${scan}" | cut -d '~' -f 2)
scanrun=$(echo "${scan}" | cut -d '~' -f 3)
if [ ${scantime} -gt ${longest_scantime} ]; then
longest_scantime=${scantime}
longest_scandate=$(echo "${scan}" | cut -d '~' -f 1)
fi
if [ ${scanrun} -eq 0 ]; then
if [ ${scantime} -lt 60 ]; then
scantimes_norm=$(bc <<< "scale=2; ${scantimes_norm}+${scantime}")
((scansfound_norm++))
else
scantimes_norm_xtr=$(bc <<< "scale=2; ${scantimes_norm_xtr}+${scantime}")
((scansfound_norm_xtr++))
fi
elif [ ${scanrun} -eq 1 ]; then
if [ ${scantime} -lt 60 ]; then
scantimes_repl=$(bc <<< "scale=2; ${scantimes_repl}+${scantime}")
((scansfound_repl++))
else
scantimes_repl_xtr=$(bc <<< "scale=2; ${scantimes_repl_xtr}+${scantime}")
((scansfound_repl_xtr++))
fi
elif [ ${scanrun} -eq 2 ]; then
if [ ${scantime} -lt 60 ]; then
scantimes_replchk=$(bc <<< "scale=2; ${scantimes_replchk}+${scantime}")
((scansfound_replchk++))
else
scantimes_replchk_xtr=$(bc <<< "scale=2; ${scantimes_replchk_xtr}+${scantime}")
((scansfound_replchk_xtr++))
fi
fi
done
scantimes_total_norm=$(expr ${scantimes_norm} + ${scantimes_norm_xtr})
scantimes_total_repl=$(expr ${scantimes_repl} + ${scantimes_repl_xtr})
scantimes_total_replchk=$(expr ${scantimes_replchk} + ${scantimes_replchk_xtr})
scantimes_total=$(expr ${scantimes_total_norm} + ${scantimes_total_repl} + ${scantimes_total_replchk})
scansfound_total_norm=$(expr ${scansfound_norm} + ${scansfound_norm_xtr})
scansfound_total_repl=$(expr ${scansfound_repl} + ${scansfound_repl_xtr})
scansfound_total_replchk=$(expr ${scansfound_replchk} + ${scansfound_replchk_xtr})
scansfound_total=$(expr ${scansfound_total_norm} + ${scansfound_total_repl} + ${scansfound_total_replchk})
[ ${scansfound_norm} -eq 0 ] && average_norm=0 || average_norm=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_norm}/${scansfound_norm}") 0)
[ ${scansfound_norm_xtr} -eq 0 ] && average_norm_xtr=0 || average_norm_xtr=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_norm_xtr}/${scansfound_norm_xtr}") 0)
[ ${scansfound_total_norm} -eq 0 ] && average_total_norm=0 || average_total_norm=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_total_norm}/${scansfound_total_norm}") 0)
[ ${scansfound_repl} -eq 0 ] && average_repl=0 || average_repl=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_repl}/${scansfound_repl}") 0)
[ ${scansfound_repl_xtr} -eq 0 ] && average_repl_xtr=0 || average_repl_xtr=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_repl_xtr}/${scansfound_repl_xtr}") 0)
[ ${scansfound_total_repl} -eq 0 ] && average_total_repl=0 || average_total_repl=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_total_repl}/${scansfound_total_repl}") 0)
[ ${scansfound_replchk} -eq 0 ] && average_replchk=0 || average_replchk=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_replchk}/${scansfound_replchk}") 0)
[ ${scansfound_replchk_xtr} -eq 0 ] && average_replchk_xtr=0 || average_replchk_xtr=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_replchk_xtr}/${scansfound_replchk_xtr}") 0)
[ ${scansfound_total_replchk} -eq 0 ] && average_total_replchk=0 || average_total_replchk=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_total_replchk}/${scansfound_total_replchk}") 0)
[ ${scansfound_total} -eq 0 ] && average_total=0 || average_total=$(ROUND_NUMBER $(bc <<< "scale=2; ${scantimes_total}/${scansfound_total}") 0)
cw=`expr $(echo "${#average_norm} ${#average_norm_xtr} ${#average_repl} ${#average_repl_xtr} ${#average_total_norm} ${#average_total_replt} ${#average_total}" | grep -oE '[0-9]+' | sort -n | tail -n 1) + 1`
average_norm_disp=$(SHOW_TIME ${average_norm} s); c=0; spcAND=''; spc1=`expr ${cw} - ${#average_norm}`; until [ $c = ${spc1} ]; do spcAND="${spcAND} "; c=`expr $c + 1`; done
average_norm_xtr_disp=$(SHOW_TIME ${average_norm_xtr} s); c=0; spcANXD=''; spc1=`expr ${cw} - ${#average_norm_xtr}`; until [ $c = ${spc1} ]; do spcANXD="${spcANXD} "; c=`expr $c + 1`; done
average_total_norm_disp=$(SHOW_TIME ${average_total_norm} s); c=0; spcATND=''; spc1=`expr ${cw} - ${#average_total_norm}`; until [ $c = ${spc1} ]; do spcATND="${spcATND} "; c=`expr $c + 1`; done
average_repl_disp=$(SHOW_TIME ${average_repl} s); c=0; spcARD=''; spc1=`expr ${cw} - ${#average_repl}`; until [ $c = ${spc1} ]; do spcARD="${spcARD} "; c=`expr $c + 1`; done
average_repl_xtr_disp=$(SHOW_TIME ${average_repl_xtr} s); c=0; spcARXD=''; spc1=`expr ${cw} - ${#average_repl_xtr}`; until [ $c = ${spc1} ]; do spcARXD="${spcARXD} "; c=`expr $c + 1`; done
average_total_repl_disp=$(SHOW_TIME ${average_total_repl} s); c=0; spcATRD=''; spc1=`expr ${cw} - ${#average_total_repl}`; until [ $c = ${spc1} ]; do spcATRD="${spcATRD} "; c=`expr $c + 1`; done
average_replchk_disp=$(SHOW_TIME ${average_replchk} s); c=0; spcARCD=''; spc1=`expr ${cw} - ${#average_replchk}`; until [ $c = ${spc1} ]; do spcARCD="${spcARCD} "; c=`expr $c + 1`; done
average_replchk_xtr_disp=$(SHOW_TIME ${average_replchk_xtr} s); c=0; spcARCXD=''; spc1=`expr ${cw} - ${#average_replchk_xtr}`; until [ $c = ${spc1} ]; do spcARCXD="${spcARCXD} "; c=`expr $c + 1`; done
average_total_replchk_disp=$(SHOW_TIME ${average_total_replchk} s); c=0; spcATRCD=''; spc1=`expr ${cw} - ${#average_total_replchk}`; until [ $c = ${spc1} ]; do spcATRCD="${spcATRCD} "; c=`expr $c + 1`; done
average_total_disp=$(SHOW_TIME ${average_total} s); c=0; spcATD=''; spc1=`expr ${cw} - ${#average_total}`; until [ $c = ${spc1} ]; do spcATD="${spcATD} "; c=`expr $c + 1`; done
lastscantime_disp=$(SHOW_TIME ${lastscantime} s)
longest_scantime_disp=$(SHOW_TIME ${longest_scantime} s)
[ -f ${NM_TMPFOLDER}/.replcheck ] && last_replcheck=$(date -d @$(stat -c %Y ${NM_TMPFOLDER}/.replcheck) "+%Y-%m-%d %H:%M:%S") || last_replcheck=""
[ -f ${NM_TMPFOLDER}/.replclean ] && last_replclean=$(date -d @$(stat -c %Y ${NM_TMPFOLDER}/.replclean) "+%Y-%m-%d %H:%M:%S") || last_replclean=""
[ -f ${NM_LOGFOLDER}/status-check.scantimes ] && last_scancheck=$(date -d @$(stat -c %Y ${NM_LOGFOLDER}/status-check.scantimes) "+%Y-%m-%d %H:%M:%S") || last_scancheck=""
echo -en "\r\033[K"
echo -e "${idsCL[LightCyan]}Scan Type Scans Average Scantime${idsCL[Default]}"
DIVIDER . lightCyan 60
echo -e "${idsCL[LightGreen]}Average for last `IDS_NUMBER_FORMAT ${scansfound_total}` ${idsCL[Green]}(${average_total}s)${spcATD}${idsCL[LightGreen]}${average_total_disp}${idsCL[Default]}"
DIVIDER . lightCyan 60
echo -e "${idsCL[Green]} Service Checks `IDS_NUMBER_FORMAT ${scansfound_total_norm}` ${idsCL[Green]}(${average_total_norm}s)${spcATND}${idsCL[LightGreen]}${average_total_norm_disp}${idsCL[Default]}"
DIVIDER . lightCyan 60
echo -e "${idsCL[Cyan]} - Normal `IDS_NUMBER_FORMAT ${scansfound_norm}` ${idsCL[Cyan]}(${average_norm}s)${spcAND}${idsCL[LightCyan]}${average_norm_disp}${idsCL[Default]}"
echo -e "${idsCL[Cyan]} - Long `IDS_NUMBER_FORMAT ${scansfound_norm_xtr}` ${idsCL[Cyan]}(${average_norm_xtr}s)${spcANXD}${idsCL[LightCyan]}${average_norm_xtr_disp}${idsCL[Default]}"
DIVIDER . lightCyan 60
echo -e "${idsCL[Green]} w/Replication `IDS_NUMBER_FORMAT ${scansfound_total_repl}` ${idsCL[Green]}(${average_total_repl}s)${spcATRD}${idsCL[LightGreen]}${average_total_repl_disp}${idsCL[Default]}"
DIVIDER . lightCyan 60
echo -e "${idsCL[Cyan]} - Normal `IDS_NUMBER_FORMAT ${scansfound_repl}` ${idsCL[Cyan]}(${average_repl}s)${spcARD}${idsCL[LightCyan]}${average_repl_disp}${idsCL[Default]}"
echo -e "${idsCL[Cyan]} - Long `IDS_NUMBER_FORMAT ${scansfound_repl_xtr}` ${idsCL[Cyan]}(${average_repl_xtr}s)${spcARXD}${idsCL[LightCyan]}${average_repl_xtr_disp}${idsCL[Default]}"
DIVIDER . lightCyan 60
echo -e "${idsCL[Green]} w/Repl. & Clean `IDS_NUMBER_FORMAT ${scansfound_total_replchk}` ${idsCL[Green]}(${average_total_replchk}s)${spcATRCD}${idsCL[LightGreen]}${average_total_replchk_disp}${idsCL[Default]}"
DIVIDER . lightCyan 60
echo -e "${idsCL[Cyan]} - Normal `IDS_NUMBER_FORMAT ${scansfound_replchk}` ${idsCL[Cyan]}(${average_replchk}s)${spcARCD}${idsCL[LightCyan]}${average_replchk_disp}${idsCL[Default]}"
echo -e "${idsCL[Cyan]} - Long `IDS_NUMBER_FORMAT ${scansfound_replchk_xtr}` ${idsCL[Cyan]}(${average_replchk_xtr}s)${spcARCXD}${idsCL[LightCyan]}${average_replchk_xtr_disp}${idsCL[Default]}"
echo
echo -e "${idsCL[LightCyan]}Longest Scan Time : ${idsCL[Green]}(${longest_scantime}s) ${idsCL[LightGreen]}${longest_scantime_disp}${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Longest Scan Date : ${idsCL[LightGreen]}${longest_scandate}${idsCL[Default]}"
echo
echo -e "${idsCL[LightCyan]}Last Scan Time : ${idsCL[Green]}(${lastscantime}s) ${idsCL[LightGreen]}${lastscantime_disp}${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Last Service Scan : ${idsCL[LightGreen]}${last_scancheck}${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Last w/Replication Scan : ${idsCL[LightGreen]}${last_replcheck}${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Last w/Repl. & Clean : ${idsCL[LightGreen]}${last_replclean}${idsCL[Default]}"
# if [ ${scansfound_total} -le 500 ]; then
if [ ${scansfound_total} -gt 20 ]; then
echo -en "\n${idsCL[LightCyan]}List out these '${scansfound_total}' scan times? (${idsCL[Yellow]}A${idsCL[LightCyan]})ll/(${idsCL[Yellow]}N${idsCL[LightCyan]})ormal/(${idsCL[Yellow]}R${idsCL[LightCyan]})epl/repl(${idsCL[Yellow]}C${idsCL[LightCyan]})lean/(${idsCL[Yellow]}L${idsCL[LightCyan]})ong [default=no] : ${idsCL[Default]}"
read -n1 choice
if [ "${choice^^}" == "A" ] || [ "${choice^^}" == "N" ] || [ "${choice^^}" == "R" ] || [ "${choice^^}" == "C" ] || [ "${choice^^}" == "L" ]; then
echo -en "\r\033[K"
gofor=1
else
gofor=0
fi
else
gofor=2
echo
fi
if [ ${gofor} -gt 0 ]; then
lastscansnum=${scansfound_total}
last_scantimes=$(tail -n ${lastscansnum} ${NM_LOGFOLDER}/status-check.scantimes)
IFS=$'\n'
read -rd '' -a last_scantimes <<<"${last_scantimes}"
unset IFS
echo -e "${idsCL[LightCyan]}Last ${lastscansnum} Scan Times\n${idsCL[Green]}------------------------------------------${idsCL[Default]}"
for scan in "${last_scantimes[@]}"; do
scandate=$(echo "${scan}" | cut -d '~' -f 1)
scantime=$(echo "${scan}" | cut -d '~' -f 2)
scanrun=$(echo "${scan}" | cut -d '~' -f 3)
if [ ${gofor} -eq 2 ] || [ "${choice^^}" == "A" ] || ([ "${choice^^}" == "N" ] && [ ${scanrun} -eq 0 ]) || ([ "${choice^^}" == "R" ] && [ ${scanrun} -eq 1 ]) || ([ "${choice^^}" == "C" ] && [ ${scanrun} -eq 2 ]) || ([ "${choice^^}" == "L" ] && ([ ${scanrun} -eq 0 ] && [ ${scantime} -gt 60 ] || [ ${scanrun} -gt 0 ] && [ ${scantime} -gt 70 ])); then
echo -en "${idsCL[White]} ${scandate} (${scantime}s) ${idsCL[Yellow]}$(SHOW_TIME ${scantime} s) "
[ ${scanrun} -eq 1 ] && echo -en "${idsCL[LightCyan]}<-- Replication Run "
[ ${scanrun} -eq 2 ] && echo -en "${idsCL[Blue]}<-- Replication & Clean Run "
[ ${scanrun} -eq 0 ] && [ ${scantime} -gt 60 ] && echo -en "${idsCL[LightRed]} <!!--[LONG RUN]--!!>"
[ ${scanrun} -gt 0 ] && [ ${scantime} -gt 70 ] && echo -en "${idsCL[LightRed]} <!!--[LONG RUN]--!!>"
echo -e "${idsCL[Default]}"
fi
done
fi
# fi
end=`date +%s`; runtime=$((end-start))
echo -e "\nRuntime: ${runtime}\n"
}
Reference in New Issue View Git Blame Copy Permalink