Update nodemgmt-scripts.sh
This commit is contained in:
@@ -27,126 +27,72 @@ STATUS() {
|
||||
echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}"
|
||||
echo
|
||||
declare -i cw; declare -i spc1; declare -i c
|
||||
if [ ! -z ${LOCAL_SERVICES+x} ];
|
||||
then NHCMD='ssh root@10.5.10.51'
|
||||
else NHCMD=''
|
||||
fi
|
||||
# for nip in "${NODE_HOSTS[@]}"; do
|
||||
# if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
|
||||
# if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
|
||||
# NCMD="ssh root@${nip}"
|
||||
# for rcheck in "${REPL_CHECK[@]}"; do
|
||||
# ${NCMD} touch ${REPL_CHECKS[${rcheck}]}/test.repl
|
||||
# done
|
||||
# fi
|
||||
# done
|
||||
|
||||
for rcheck in "${REPL_CHECK[@]}"; do
|
||||
${NHCMD} rm -f ${FOLDER}/test.repl
|
||||
daterun=`date +%Y-%m-%d-%H-%M-%S`
|
||||
if [ "${NHCMD}" = "" ]; then
|
||||
echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
|
||||
yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
|
||||
else
|
||||
${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
|
||||
yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
|
||||
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then
|
||||
if [ ! -z ${LOCAL_SERVICES+x} ];
|
||||
then NHCMD='ssh root@10.5.10.51'
|
||||
else NHCMD=''
|
||||
fi
|
||||
done
|
||||
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}"
|
||||
DIVIDER; echo -e "${idsST[Reset]}"
|
||||
if [ ! -z ${LOCAL_SERVICES+x} ]; then
|
||||
lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
|
||||
uptime=`uptime -p`
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
for srvc in "${LOCAL_SERVICES[@]}"
|
||||
do
|
||||
c=0; cw=22; spc=''
|
||||
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
||||
if [ $(pgrep ${srvc} | wc -l) -gt "0" ]; then
|
||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||
rm -f ${FOLDER}/localhost-${srvc}.down
|
||||
for rcheck in "${REPL_CHECK[@]}"; do
|
||||
${NHCMD} rm -f ${FOLDER}/test.repl
|
||||
daterun=`date +%Y-%m-%d-%H-%M-%S`
|
||||
if [ "${NHCMD}" = "" ]; then
|
||||
echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
|
||||
yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
|
||||
else
|
||||
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||
touch ${FOLDER}/localhost-${srvc}.down
|
||||
${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
|
||||
yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
fi
|
||||
nid=1
|
||||
for nip in "${NODE_HOSTS[@]}"; do
|
||||
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
||||
else NCMD="ssh root@${nip}"; LH=''
|
||||
fi
|
||||
uptime=`${NCMD} uptime -p`
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
checkhost=$(CHECK_HOST ${nip})
|
||||
if [ "${checkhost}" != "false" ]; then
|
||||
rm -f ${FOLDER}/${nip}.down
|
||||
for srvc in "${NODE_SERVICES[@]}"
|
||||
|
||||
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "services" ]; then
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}"
|
||||
DIVIDER; echo -e "${idsST[Reset]}"
|
||||
if [ ! -z ${LOCAL_SERVICES+x} ]; then
|
||||
lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
|
||||
uptime=`uptime -p`
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
for srvc in "${LOCAL_SERVICES[@]}"
|
||||
do
|
||||
c=0; cw=22; spc=''
|
||||
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
||||
if [ $(${NCMD} pgrep ${srvc} | wc -l) -gt "0" ]; then
|
||||
if [ $(pgrep ${srvc} | wc -l) -gt "0" ]; then
|
||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||
rm -f ${FOLDER}/${nip}-${srvc}.down
|
||||
rm -f ${FOLDER}/localhost-${srvc}.down
|
||||
else
|
||||
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||
touch ${FOLDER}/${nip}-${srvc}.down
|
||||
touch ${FOLDER}/localhost-${srvc}.down
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
|
||||
touch ${FOLDER}/${nip}.down
|
||||
echo
|
||||
fi
|
||||
echo
|
||||
nid=`expr $nid + 1`
|
||||
done
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
|
||||
DIVIDER; echo -e "${idsST[Reset]}"
|
||||
nid=1
|
||||
for nip in "${NODE_HOSTS[@]}"; do
|
||||
if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
|
||||
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
|
||||
nid=1
|
||||
for nip in "${NODE_HOSTS[@]}"; do
|
||||
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
||||
else NCMD="ssh root@${nip}"; LH=''
|
||||
fi
|
||||
uptime=`${NCMD} uptime -p`
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
checkhost=$(CHECK_HOST ${nip})
|
||||
if [ "${checkhost}" != "false" ]; then
|
||||
rm -f ${FOLDER}/${nip}.down
|
||||
for rcheck in "${REPL_CHECK[@]}"; do
|
||||
for srvc in "${NODE_SERVICES[@]}"
|
||||
do
|
||||
c=0; cw=22; spc=''
|
||||
spc1=${cw}-${#REPL_DESC[${rcheck}]}
|
||||
timeout=`date --date='5 seconds' +%s`
|
||||
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
|
||||
checked=false
|
||||
until [ "${checked}" = "" ]; do
|
||||
if [ "${NHCMD}" = "" ]; then
|
||||
checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
|
||||
else
|
||||
checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
|
||||
fi
|
||||
#cdate=`date +%s`
|
||||
if [ "`date +%s`" -gt "$timeout" ]; then
|
||||
timeout=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "${timeout}" != "true" ]; then
|
||||
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
|
||||
rm -f ${FOLDER}/${nip}-${rcheck}.down
|
||||
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
||||
if [ $(${NCMD} pgrep ${srvc} | wc -l) -gt "0" ]; then
|
||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||
rm -f ${FOLDER}/${nip}-${srvc}.down
|
||||
else
|
||||
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
||||
touch ${FOLDER}/${nip}-${rcheck}.down
|
||||
#echo "Timeout occurred in waiting for replication between primary node and iDS-Node${nid} (${nip}) for the replicated folder of '${rcheck}'." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
|
||||
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||
touch ${FOLDER}/${nip}-${srvc}.down
|
||||
fi
|
||||
done
|
||||
else
|
||||
@@ -154,9 +100,60 @@ STATUS() {
|
||||
touch ${FOLDER}/${nip}.down
|
||||
fi
|
||||
echo
|
||||
fi
|
||||
nid=`expr $nid + 1`
|
||||
done
|
||||
nid=`expr $nid + 1`
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
|
||||
DIVIDER; echo -e "${idsST[Reset]}"
|
||||
nid=1
|
||||
for nip in "${NODE_HOSTS[@]}"; do
|
||||
if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
|
||||
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
checkhost=$(CHECK_HOST ${nip})
|
||||
if [ "${checkhost}" != "false" ]; then
|
||||
rm -f ${FOLDER}/${nip}.down
|
||||
for rcheck in "${REPL_CHECK[@]}"; do
|
||||
c=0; cw=22; spc=''
|
||||
spc1=${cw}-${#REPL_DESC[${rcheck}]}
|
||||
timeout=`date --date='5 seconds' +%s`
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
|
||||
checked=false
|
||||
until [ "${checked}" = "" ]; do
|
||||
if [ "${NHCMD}" = "" ]; then
|
||||
checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
|
||||
else
|
||||
checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
|
||||
fi
|
||||
#cdate=`date +%s`
|
||||
if [ "`date +%s`" -gt "$timeout" ]; then
|
||||
timeout=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "${timeout}" != "true" ]; then
|
||||
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
|
||||
rm -f ${FOLDER}/${nip}-${rcheck}.down
|
||||
else
|
||||
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
||||
touch ${FOLDER}/${nip}-${rcheck}.down
|
||||
#echo "Timeout occurred in waiting for replication between primary node and iDS-Node${nid} (${nip}) for the replicated folder of '${rcheck}'." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
|
||||
touch ${FOLDER}/${nip}.down
|
||||
fi
|
||||
echo
|
||||
fi
|
||||
nid=`expr $nid + 1`
|
||||
done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
||||
|
||||
Reference in New Issue
Block a user