update
This commit is contained in:
@@ -133,6 +133,293 @@ STATUSNEW(){
|
|||||||
|
|
||||||
for NTYPE in "${ntypesel[@]}"; do
|
for NTYPE in "${ntypesel[@]}"; do
|
||||||
|
|
||||||
|
STATUS_NODE
|
||||||
|
|
||||||
|
done
|
||||||
|
|
||||||
|
fi
|
||||||
|
########################
|
||||||
|
## REPLICATION CHECK
|
||||||
|
########################
|
||||||
|
|
||||||
|
if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsST[Bold]}"; DIVIDER
|
||||||
|
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
|
||||||
|
DIVIDER; echo -e "${idsST[Reset]}"
|
||||||
|
echo -en " ${idsCL[LightCyan]}Starting processes to collect/monitor replication status : "
|
||||||
|
fi
|
||||||
|
for NTYPE in "${ntypesel[@]}"; do
|
||||||
|
PH=${NTYPE}_HOSTS[0]
|
||||||
|
PH_CMD="ssh root@${!PH}"
|
||||||
|
var=${NTYPE}_REPL_CHECK[@]
|
||||||
|
if [ ! -z ${!var+x} ]; then
|
||||||
|
var=${NTYPE}_HOSTS[@]
|
||||||
|
for nip in "${!var}"; do
|
||||||
|
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
|
||||||
|
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
||||||
|
var2=${NTYPE}_REPL_CHECK[@]
|
||||||
|
for rcheck in "${!var2}"; do
|
||||||
|
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
||||||
|
|
||||||
|
for NTYPE in "${ntypesel[@]}"; do
|
||||||
|
PH=${NTYPE}_HOSTS[0]
|
||||||
|
PH_CMD="ssh root@${!PH}"
|
||||||
|
var=${NTYPE}_REPL_CHECK[@]
|
||||||
|
if [ ! -z ${!var+x} ]; then
|
||||||
|
var=${NTYPE}_HOSTS[@]
|
||||||
|
for nip in "${!var}"; do
|
||||||
|
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
|
||||||
|
DIVIDER false green
|
||||||
|
fi
|
||||||
|
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
||||||
|
var2=${NTYPE}_REPL_CHECK[@]
|
||||||
|
for rcheck in "${!var2}"; do
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
if [ "${ST_ACTION}" != "report" ]; then
|
||||||
|
c=0; cw=${PRI_CW}; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
|
else
|
||||||
|
spc=' '
|
||||||
|
fi
|
||||||
|
echo -en " ${NM_REPL_DESC[${rcheck}]}${spc}: "
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
checked=""
|
||||||
|
until [ "${checked}" != "" ]; do
|
||||||
|
if [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
|
||||||
|
checked=good
|
||||||
|
elif [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
|
||||||
|
checked=timeout
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "${checked}" == "timeout" ]; then
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
||||||
|
if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
|
||||||
|
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ]; then
|
||||||
|
mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
|
||||||
|
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
|
||||||
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
|
||||||
|
else
|
||||||
|
snderrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down))
|
||||||
|
# echo "HERE: $snderrtime"
|
||||||
|
if [ ${snderrtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
|
||||||
|
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime))
|
||||||
|
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
|
||||||
|
It has been down for $(SHOW_TIME ${toterrtime})" 1
|
||||||
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, has now been down for: $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
elif [ "${checked}" == "good" ]; then
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}"
|
||||||
|
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
|
||||||
|
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ]; then
|
||||||
|
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime))
|
||||||
|
SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
|
||||||
|
It was down for $(SHOW_TIME ${toterrtime})"
|
||||||
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}
|
||||||
|
fi
|
||||||
|
rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl*
|
||||||
|
fi
|
||||||
|
ssh -q root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;'
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
else
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
[ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
|
||||||
|
[ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
##########################
|
||||||
|
# REMOVE REPL CHECK FILES
|
||||||
|
##########################
|
||||||
|
if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
#NHCMD="ssh root@${NM_HOSTS['MYSQL'][0]}"
|
||||||
|
#LBHCMD="ssh root@${NM_HOSTS['LB'][0]}"
|
||||||
|
#WHCMD="ssh root@${NM_HOSTS['WEB'][0]}"
|
||||||
|
for NTYPE in "${ntypesel[@]}"; do
|
||||||
|
PH=${NTYPE}_HOSTS[0]
|
||||||
|
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
|
||||||
|
PH_CMD="ssh root@${!PH}"
|
||||||
|
var=${NTYPE}_REPL_CHECK[@]
|
||||||
|
if [ ! -z ${!var+x} ]; then
|
||||||
|
for rcheck in "${!var}"; do
|
||||||
|
${PH_CMD} rm -f ${NM_FOLDER}/test.repl
|
||||||
|
daterun=`date +%Y-%m-%d-%H-%M-%S`
|
||||||
|
if [ "${PH_CMD}" = "" ]; then
|
||||||
|
rm -f ${NM_FOLDER}/test.repl &
|
||||||
|
rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
|
||||||
|
else
|
||||||
|
${PH_CMD} rm -f ${NM_FOLDER}/test.repl &
|
||||||
|
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
|
||||||
|
fi
|
||||||
|
rm -f ${NM_TMPFOLDER}/repl.${rcheck}.*
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
rm -Rf ${NM_FOLDER}/test.repl
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}"
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
########################
|
||||||
|
## FREE SPACE CHECK
|
||||||
|
########################
|
||||||
|
#if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then
|
||||||
|
if [ "${ST_ACTION}" = "freespace" ]; then
|
||||||
|
if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then
|
||||||
|
for NTYPE in "${ntypesel[@]}"; do
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsST[Bold]}"; DIVIDER
|
||||||
|
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node(s) Free Space Scan${idsCL[Default]}"
|
||||||
|
DIVIDER; echo -e "${idsST[Reset]}"
|
||||||
|
fi
|
||||||
|
var=${NTYPE}_HOSTS[@]
|
||||||
|
for nip in "${!var}"; do
|
||||||
|
if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
||||||
|
else NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH=''
|
||||||
|
fi
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
||||||
|
DIVIDER false green
|
||||||
|
fi
|
||||||
|
|
||||||
|
[ "${ST_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
|
||||||
|
declare -A partitions
|
||||||
|
DRIVEINFO=$(ssh -o ConnectTimeout=3 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
|
||||||
|
DRIVEINFO=(${DRIVEINFO})
|
||||||
|
if [ "${2}" != "report" ]; then
|
||||||
|
echo -en "\e[1A"
|
||||||
|
echo -e "\e[0K\r"
|
||||||
|
fi
|
||||||
|
|
||||||
|
NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
|
||||||
|
for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
|
||||||
|
ii=$((${i}*3))
|
||||||
|
pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
|
||||||
|
pname=${pname#*vg-}
|
||||||
|
if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then
|
||||||
|
freespace=${DRIVEINFO[$((${ii}+2))]//M/}
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
c=0; cw=20; spc=''
|
||||||
|
spc1=${cw}-${#pname}
|
||||||
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
|
echo -en " ${pname}$spc: "
|
||||||
|
fi
|
||||||
|
if [[ ${freespace} -gt 1024 ]]; then
|
||||||
|
fsgb=$(bc <<< "scale=2; ${freespace}/1024")
|
||||||
|
fsdsp="${fsgb} GB"
|
||||||
|
else
|
||||||
|
fsdsp="${freespace} MB"
|
||||||
|
fi
|
||||||
|
if [ "${freespace}" -le "1024" ]; then
|
||||||
|
fs_status='error'
|
||||||
|
fs_status_color='Red'
|
||||||
|
SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free" 1
|
||||||
|
elif [ "${freespace}" -le "5120" ]; then
|
||||||
|
fs_status='warn'
|
||||||
|
fs_status_color='Yellow'
|
||||||
|
SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free"
|
||||||
|
else
|
||||||
|
fs_status=''
|
||||||
|
fs_status_color='Green'
|
||||||
|
fi
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# for partition in "${partitions3[@]}"; do
|
||||||
|
# if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then
|
||||||
|
# pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
|
||||||
|
# pname=${pname#*vg-}
|
||||||
|
# if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
# c=0; cw=20; spc=''
|
||||||
|
# spc1=${cw}-${#pname}
|
||||||
|
# until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
|
# echo -en " ${pname}$spc: "
|
||||||
|
# fi
|
||||||
|
#
|
||||||
|
# # [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition}
|
||||||
|
# # freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1
|
||||||
|
#
|
||||||
|
# if [ "${freespace}" -le "1000" ]; then
|
||||||
|
# fs_status='error'
|
||||||
|
# fs_status_color='Red'
|
||||||
|
# SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free" 1
|
||||||
|
#
|
||||||
|
# elif [ "${freespace}" -le "5000" ]; then
|
||||||
|
# fs_status='warn'
|
||||||
|
# fs_status_color='Yellow'
|
||||||
|
# SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free"
|
||||||
|
#
|
||||||
|
# else
|
||||||
|
# fs_status=''
|
||||||
|
# fs_status_color='Green'
|
||||||
|
#
|
||||||
|
# fi
|
||||||
|
# if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
# if [[ ${freespace} -gt 1000 ]]; then
|
||||||
|
# fsgb=$(bc <<< "scale=2; ${freespace}/1000")
|
||||||
|
# fsdsp="${fsgb} GB"
|
||||||
|
# else
|
||||||
|
# fsdsp="${freespace} MB"
|
||||||
|
# fi
|
||||||
|
# echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
|
||||||
|
#
|
||||||
|
# fi
|
||||||
|
# fi
|
||||||
|
# done
|
||||||
|
|
||||||
|
echo
|
||||||
|
done
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
########################
|
||||||
|
########################
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo
|
||||||
|
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
|
||||||
|
DIVIDER true
|
||||||
|
ENTER2CONTINUE
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
end=`date +%s`
|
||||||
|
runtime=$((end-start))
|
||||||
|
echo -e "Runtime: ${runtime}\n"
|
||||||
|
[ "${ST_ACTION}" == "report" ] && echo ${runtime} >> ${NM_LOGFOLDER}/status-check.scantimes
|
||||||
|
}
|
||||||
|
|
||||||
|
STATUS_NODE(){
|
||||||
dockers=${NTYPE}_DOCKERS_CHECK[@]
|
dockers=${NTYPE}_DOCKERS_CHECK[@]
|
||||||
hosts=${NTYPE}_HOSTS[@]
|
hosts=${NTYPE}_HOSTS[@]
|
||||||
|
|
||||||
@@ -453,287 +740,4 @@ STATUSNEW(){
|
|||||||
done
|
done
|
||||||
|
|
||||||
fi #GOFORCHECK
|
fi #GOFORCHECK
|
||||||
|
|
||||||
done
|
|
||||||
|
|
||||||
fi
|
|
||||||
########################
|
|
||||||
## REPLICATION CHECK
|
|
||||||
########################
|
|
||||||
|
|
||||||
if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsST[Bold]}"; DIVIDER
|
|
||||||
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
|
|
||||||
DIVIDER; echo -e "${idsST[Reset]}"
|
|
||||||
echo -en " ${idsCL[LightCyan]}Starting processes to collect/monitor replication status : "
|
|
||||||
fi
|
|
||||||
for NTYPE in "${ntypesel[@]}"; do
|
|
||||||
PH=${NTYPE}_HOSTS[0]
|
|
||||||
PH_CMD="ssh root@${!PH}"
|
|
||||||
var=${NTYPE}_REPL_CHECK[@]
|
|
||||||
if [ ! -z ${!var+x} ]; then
|
|
||||||
var=${NTYPE}_HOSTS[@]
|
|
||||||
for nip in "${!var}"; do
|
|
||||||
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
|
|
||||||
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
|
||||||
var2=${NTYPE}_REPL_CHECK[@]
|
|
||||||
for rcheck in "${!var2}"; do
|
|
||||||
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
||||||
|
|
||||||
for NTYPE in "${ntypesel[@]}"; do
|
|
||||||
PH=${NTYPE}_HOSTS[0]
|
|
||||||
PH_CMD="ssh root@${!PH}"
|
|
||||||
var=${NTYPE}_REPL_CHECK[@]
|
|
||||||
if [ ! -z ${!var+x} ]; then
|
|
||||||
var=${NTYPE}_HOSTS[@]
|
|
||||||
for nip in "${!var}"; do
|
|
||||||
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
|
|
||||||
DIVIDER false green
|
|
||||||
fi
|
|
||||||
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
|
||||||
var2=${NTYPE}_REPL_CHECK[@]
|
|
||||||
for rcheck in "${!var2}"; do
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
if [ "${ST_ACTION}" != "report" ]; then
|
|
||||||
c=0; cw=${PRI_CW}; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
||||||
else
|
|
||||||
spc=' '
|
|
||||||
fi
|
|
||||||
echo -en " ${NM_REPL_DESC[${rcheck}]}${spc}: "
|
|
||||||
|
|
||||||
fi
|
|
||||||
|
|
||||||
checked=""
|
|
||||||
until [ "${checked}" != "" ]; do
|
|
||||||
if [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
|
|
||||||
checked=good
|
|
||||||
elif [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
|
|
||||||
checked=timeout
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ "${checked}" == "timeout" ]; then
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
|
||||||
if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
|
|
||||||
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ]; then
|
|
||||||
mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
|
|
||||||
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
|
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
|
|
||||||
else
|
|
||||||
snderrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down))
|
|
||||||
# echo "HERE: $snderrtime"
|
|
||||||
if [ ${snderrtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
|
|
||||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime))
|
|
||||||
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
|
|
||||||
It has been down for $(SHOW_TIME ${toterrtime})" 1
|
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, has now been down for: $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
elif [ "${checked}" == "good" ]; then
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}"
|
|
||||||
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
|
|
||||||
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ]; then
|
|
||||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime))
|
|
||||||
SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
|
|
||||||
It was down for $(SHOW_TIME ${toterrtime})"
|
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}
|
|
||||||
fi
|
|
||||||
rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl*
|
|
||||||
fi
|
|
||||||
ssh -q root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;'
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
else
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
[ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
|
|
||||||
[ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo
|
|
||||||
|
|
||||||
fi
|
|
||||||
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
##########################
|
|
||||||
# REMOVE REPL CHECK FILES
|
|
||||||
##########################
|
|
||||||
if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
#NHCMD="ssh root@${NM_HOSTS['MYSQL'][0]}"
|
|
||||||
#LBHCMD="ssh root@${NM_HOSTS['LB'][0]}"
|
|
||||||
#WHCMD="ssh root@${NM_HOSTS['WEB'][0]}"
|
|
||||||
for NTYPE in "${ntypesel[@]}"; do
|
|
||||||
PH=${NTYPE}_HOSTS[0]
|
|
||||||
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
|
|
||||||
PH_CMD="ssh root@${!PH}"
|
|
||||||
var=${NTYPE}_REPL_CHECK[@]
|
|
||||||
if [ ! -z ${!var+x} ]; then
|
|
||||||
for rcheck in "${!var}"; do
|
|
||||||
${PH_CMD} rm -f ${NM_FOLDER}/test.repl
|
|
||||||
daterun=`date +%Y-%m-%d-%H-%M-%S`
|
|
||||||
if [ "${PH_CMD}" = "" ]; then
|
|
||||||
rm -f ${NM_FOLDER}/test.repl &
|
|
||||||
rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
|
|
||||||
else
|
|
||||||
${PH_CMD} rm -f ${NM_FOLDER}/test.repl &
|
|
||||||
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
|
|
||||||
fi
|
|
||||||
rm -f ${NM_TMPFOLDER}/repl.${rcheck}.*
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
rm -Rf ${NM_FOLDER}/test.repl
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}"
|
|
||||||
echo
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
########################
|
|
||||||
## FREE SPACE CHECK
|
|
||||||
########################
|
|
||||||
#if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then
|
|
||||||
if [ "${ST_ACTION}" = "freespace" ]; then
|
|
||||||
if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then
|
|
||||||
for NTYPE in "${ntypesel[@]}"; do
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsST[Bold]}"; DIVIDER
|
|
||||||
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node(s) Free Space Scan${idsCL[Default]}"
|
|
||||||
DIVIDER; echo -e "${idsST[Reset]}"
|
|
||||||
fi
|
|
||||||
var=${NTYPE}_HOSTS[@]
|
|
||||||
for nip in "${!var}"; do
|
|
||||||
if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
|
||||||
else NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH=''
|
|
||||||
fi
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
|
||||||
DIVIDER false green
|
|
||||||
fi
|
|
||||||
|
|
||||||
[ "${ST_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
|
|
||||||
declare -A partitions
|
|
||||||
DRIVEINFO=$(ssh -o ConnectTimeout=3 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
|
|
||||||
DRIVEINFO=(${DRIVEINFO})
|
|
||||||
if [ "${2}" != "report" ]; then
|
|
||||||
echo -en "\e[1A"
|
|
||||||
echo -e "\e[0K\r"
|
|
||||||
fi
|
|
||||||
|
|
||||||
NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
|
|
||||||
for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
|
|
||||||
ii=$((${i}*3))
|
|
||||||
pname=`echo "${DRIVEINFO[${ii}]}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
|
|
||||||
pname=${pname#*vg-}
|
|
||||||
if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then
|
|
||||||
freespace=${DRIVEINFO[$((${ii}+2))]//M/}
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
c=0; cw=20; spc=''
|
|
||||||
spc1=${cw}-${#pname}
|
|
||||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
||||||
echo -en " ${pname}$spc: "
|
|
||||||
fi
|
|
||||||
if [[ ${freespace} -gt 1024 ]]; then
|
|
||||||
fsgb=$(bc <<< "scale=2; ${freespace}/1024")
|
|
||||||
fsdsp="${fsgb} GB"
|
|
||||||
else
|
|
||||||
fsdsp="${freespace} MB"
|
|
||||||
fi
|
|
||||||
if [ "${freespace}" -le "1024" ]; then
|
|
||||||
fs_status='error'
|
|
||||||
fs_status_color='Red'
|
|
||||||
SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free" 1
|
|
||||||
elif [ "${freespace}" -le "5120" ]; then
|
|
||||||
fs_status='warn'
|
|
||||||
fs_status_color='Yellow'
|
|
||||||
SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${fsdsp} free"
|
|
||||||
else
|
|
||||||
fs_status=''
|
|
||||||
fs_status_color='Green'
|
|
||||||
fi
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# for partition in "${partitions3[@]}"; do
|
|
||||||
# if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then
|
|
||||||
# pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
|
|
||||||
# pname=${pname#*vg-}
|
|
||||||
# if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
# c=0; cw=20; spc=''
|
|
||||||
# spc1=${cw}-${#pname}
|
|
||||||
# until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
||||||
# echo -en " ${pname}$spc: "
|
|
||||||
# fi
|
|
||||||
#
|
|
||||||
# # [[ "${partition}" == *"root"* ]] && prt="/" || prt=${partition}
|
|
||||||
# # freespace=`${NCMD} df -hPBM ${prt} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1
|
|
||||||
#
|
|
||||||
# if [ "${freespace}" -le "1000" ]; then
|
|
||||||
# fs_status='error'
|
|
||||||
# fs_status_color='Red'
|
|
||||||
# SENDNOTICE "Free Space Critical: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free" 1
|
|
||||||
#
|
|
||||||
# elif [ "${freespace}" -le "5000" ]; then
|
|
||||||
# fs_status='warn'
|
|
||||||
# fs_status_color='Yellow'
|
|
||||||
# SENDNOTICE "Free Space Warning: '${NM_HOSTNAMES[${nip}]}[${nip}]'" "${partition} : ${freespace} MB free"
|
|
||||||
#
|
|
||||||
# else
|
|
||||||
# fs_status=''
|
|
||||||
# fs_status_color='Green'
|
|
||||||
#
|
|
||||||
# fi
|
|
||||||
# if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
# if [[ ${freespace} -gt 1000 ]]; then
|
|
||||||
# fsgb=$(bc <<< "scale=2; ${freespace}/1000")
|
|
||||||
# fsdsp="${fsgb} GB"
|
|
||||||
# else
|
|
||||||
# fsdsp="${freespace} MB"
|
|
||||||
# fi
|
|
||||||
# echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
|
|
||||||
#
|
|
||||||
# fi
|
|
||||||
# fi
|
|
||||||
# done
|
|
||||||
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
########################
|
|
||||||
########################
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo
|
|
||||||
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
|
|
||||||
DIVIDER true
|
|
||||||
ENTER2CONTINUE
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
end=`date +%s`
|
|
||||||
runtime=$((end-start))
|
|
||||||
echo -e "Runtime: ${runtime}\n"
|
|
||||||
[ "${ST_ACTION}" == "report" ] && echo ${runtime} >> ${NM_LOGFOLDER}/status-check.scantimes
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -864,6 +864,9 @@ GUI(){
|
|||||||
dl)
|
dl)
|
||||||
DOWNLIST ${2}
|
DOWNLIST ${2}
|
||||||
;;
|
;;
|
||||||
|
statusnew)
|
||||||
|
STATUSNEW
|
||||||
|
;;
|
||||||
status)
|
status)
|
||||||
if [ "${2}" == "freespace" ]; then
|
if [ "${2}" == "freespace" ]; then
|
||||||
STATUS freespace ${3}
|
STATUS freespace ${3}
|
||||||
|
|||||||
Reference in New Issue
Block a user