Update nodemgmt-scripts.sh
This commit is contained in:
@@ -27,126 +27,72 @@ STATUS() {
|
|||||||
echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}"
|
echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}"
|
||||||
echo
|
echo
|
||||||
declare -i cw; declare -i spc1; declare -i c
|
declare -i cw; declare -i spc1; declare -i c
|
||||||
if [ ! -z ${LOCAL_SERVICES+x} ];
|
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then
|
||||||
then NHCMD='ssh root@10.5.10.51'
|
if [ ! -z ${LOCAL_SERVICES+x} ];
|
||||||
else NHCMD=''
|
then NHCMD='ssh root@10.5.10.51'
|
||||||
fi
|
else NHCMD=''
|
||||||
# for nip in "${NODE_HOSTS[@]}"; do
|
|
||||||
# if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
|
|
||||||
# if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
|
|
||||||
# NCMD="ssh root@${nip}"
|
|
||||||
# for rcheck in "${REPL_CHECK[@]}"; do
|
|
||||||
# ${NCMD} touch ${REPL_CHECKS[${rcheck}]}/test.repl
|
|
||||||
# done
|
|
||||||
# fi
|
|
||||||
# done
|
|
||||||
|
|
||||||
for rcheck in "${REPL_CHECK[@]}"; do
|
|
||||||
${NHCMD} rm -f ${FOLDER}/test.repl
|
|
||||||
daterun=`date +%Y-%m-%d-%H-%M-%S`
|
|
||||||
if [ "${NHCMD}" = "" ]; then
|
|
||||||
echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
|
|
||||||
yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
|
|
||||||
else
|
|
||||||
${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
|
|
||||||
yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
|
|
||||||
fi
|
fi
|
||||||
done
|
for rcheck in "${REPL_CHECK[@]}"; do
|
||||||
|
${NHCMD} rm -f ${FOLDER}/test.repl
|
||||||
echo -e "${idsST[Bold]}"; DIVIDER
|
daterun=`date +%Y-%m-%d-%H-%M-%S`
|
||||||
echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}"
|
if [ "${NHCMD}" = "" ]; then
|
||||||
DIVIDER; echo -e "${idsST[Reset]}"
|
echo -e "iDS-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
|
||||||
if [ ! -z ${LOCAL_SERVICES+x} ]; then
|
yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
|
||||||
lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
|
|
||||||
uptime=`uptime -p`
|
|
||||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
|
|
||||||
DIVIDER false green
|
|
||||||
for srvc in "${LOCAL_SERVICES[@]}"
|
|
||||||
do
|
|
||||||
c=0; cw=22; spc=''
|
|
||||||
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
|
||||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
||||||
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
|
||||||
if [ $(pgrep ${srvc} | wc -l) -gt "0" ]; then
|
|
||||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
|
||||||
rm -f ${FOLDER}/localhost-${srvc}.down
|
|
||||||
else
|
else
|
||||||
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
${NHCMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
|
||||||
touch ${FOLDER}/localhost-${srvc}.down
|
yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
echo
|
|
||||||
fi
|
fi
|
||||||
nid=1
|
|
||||||
for nip in "${NODE_HOSTS[@]}"; do
|
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "services" ]; then
|
||||||
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
echo -e "${idsST[Bold]}"; DIVIDER
|
||||||
else NCMD="ssh root@${nip}"; LH=''
|
echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}"
|
||||||
fi
|
DIVIDER; echo -e "${idsST[Reset]}"
|
||||||
uptime=`${NCMD} uptime -p`
|
if [ ! -z ${LOCAL_SERVICES+x} ]; then
|
||||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}"
|
lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
|
||||||
DIVIDER false green
|
uptime=`uptime -p`
|
||||||
checkhost=$(CHECK_HOST ${nip})
|
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
|
||||||
if [ "${checkhost}" != "false" ]; then
|
DIVIDER false green
|
||||||
rm -f ${FOLDER}/${nip}.down
|
for srvc in "${LOCAL_SERVICES[@]}"
|
||||||
for srvc in "${NODE_SERVICES[@]}"
|
|
||||||
do
|
do
|
||||||
c=0; cw=22; spc=''
|
c=0; cw=22; spc=''
|
||||||
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
||||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
||||||
if [ $(${NCMD} pgrep ${srvc} | wc -l) -gt "0" ]; then
|
if [ $(pgrep ${srvc} | wc -l) -gt "0" ]; then
|
||||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||||
rm -f ${FOLDER}/${nip}-${srvc}.down
|
rm -f ${FOLDER}/localhost-${srvc}.down
|
||||||
else
|
else
|
||||||
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||||
touch ${FOLDER}/${nip}-${srvc}.down
|
touch ${FOLDER}/localhost-${srvc}.down
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
else
|
echo
|
||||||
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
|
|
||||||
touch ${FOLDER}/${nip}.down
|
|
||||||
fi
|
fi
|
||||||
echo
|
nid=1
|
||||||
nid=`expr $nid + 1`
|
for nip in "${NODE_HOSTS[@]}"; do
|
||||||
done
|
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
||||||
echo -e "${idsST[Bold]}"; DIVIDER
|
else NCMD="ssh root@${nip}"; LH=''
|
||||||
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
|
fi
|
||||||
DIVIDER; echo -e "${idsST[Reset]}"
|
uptime=`${NCMD} uptime -p`
|
||||||
nid=1
|
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}"
|
||||||
for nip in "${NODE_HOSTS[@]}"; do
|
|
||||||
if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
|
|
||||||
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
|
|
||||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
|
|
||||||
DIVIDER false green
|
DIVIDER false green
|
||||||
checkhost=$(CHECK_HOST ${nip})
|
checkhost=$(CHECK_HOST ${nip})
|
||||||
if [ "${checkhost}" != "false" ]; then
|
if [ "${checkhost}" != "false" ]; then
|
||||||
rm -f ${FOLDER}/${nip}.down
|
rm -f ${FOLDER}/${nip}.down
|
||||||
for rcheck in "${REPL_CHECK[@]}"; do
|
for srvc in "${NODE_SERVICES[@]}"
|
||||||
|
do
|
||||||
c=0; cw=22; spc=''
|
c=0; cw=22; spc=''
|
||||||
spc1=${cw}-${#REPL_DESC[${rcheck}]}
|
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
||||||
timeout=`date --date='5 seconds' +%s`
|
|
||||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
|
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
||||||
checked=false
|
if [ $(${NCMD} pgrep ${srvc} | wc -l) -gt "0" ]; then
|
||||||
until [ "${checked}" = "" ]; do
|
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||||
if [ "${NHCMD}" = "" ]; then
|
rm -f ${FOLDER}/${nip}-${srvc}.down
|
||||||
checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
|
|
||||||
else
|
|
||||||
checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
|
|
||||||
fi
|
|
||||||
#cdate=`date +%s`
|
|
||||||
if [ "`date +%s`" -gt "$timeout" ]; then
|
|
||||||
timeout=true
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [ "${timeout}" != "true" ]; then
|
|
||||||
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
|
|
||||||
rm -f ${FOLDER}/${nip}-${rcheck}.down
|
|
||||||
else
|
else
|
||||||
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||||
touch ${FOLDER}/${nip}-${rcheck}.down
|
touch ${FOLDER}/${nip}-${srvc}.down
|
||||||
#echo "Timeout occurred in waiting for replication between primary node and iDS-Node${nid} (${nip}) for the replicated folder of '${rcheck}'." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
|
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
@@ -154,9 +100,60 @@ STATUS() {
|
|||||||
touch ${FOLDER}/${nip}.down
|
touch ${FOLDER}/${nip}.down
|
||||||
fi
|
fi
|
||||||
echo
|
echo
|
||||||
fi
|
nid=`expr $nid + 1`
|
||||||
nid=`expr $nid + 1`
|
done
|
||||||
done
|
fi
|
||||||
|
|
||||||
|
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then
|
||||||
|
echo -e "${idsST[Bold]}"; DIVIDER
|
||||||
|
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
|
||||||
|
DIVIDER; echo -e "${idsST[Reset]}"
|
||||||
|
nid=1
|
||||||
|
for nip in "${NODE_HOSTS[@]}"; do
|
||||||
|
if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
|
||||||
|
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
|
||||||
|
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
|
||||||
|
DIVIDER false green
|
||||||
|
checkhost=$(CHECK_HOST ${nip})
|
||||||
|
if [ "${checkhost}" != "false" ]; then
|
||||||
|
rm -f ${FOLDER}/${nip}.down
|
||||||
|
for rcheck in "${REPL_CHECK[@]}"; do
|
||||||
|
c=0; cw=22; spc=''
|
||||||
|
spc1=${cw}-${#REPL_DESC[${rcheck}]}
|
||||||
|
timeout=`date --date='5 seconds' +%s`
|
||||||
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
|
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
|
||||||
|
checked=false
|
||||||
|
until [ "${checked}" = "" ]; do
|
||||||
|
if [ "${NHCMD}" = "" ]; then
|
||||||
|
checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
|
||||||
|
else
|
||||||
|
checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
|
||||||
|
fi
|
||||||
|
#cdate=`date +%s`
|
||||||
|
if [ "`date +%s`" -gt "$timeout" ]; then
|
||||||
|
timeout=true
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "${timeout}" != "true" ]; then
|
||||||
|
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
|
||||||
|
rm -f ${FOLDER}/${nip}-${rcheck}.down
|
||||||
|
else
|
||||||
|
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
||||||
|
touch ${FOLDER}/${nip}-${rcheck}.down
|
||||||
|
#echo "Timeout occurred in waiting for replication between primary node and iDS-Node${nid} (${nip}) for the replicated folder of '${rcheck}'." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
else
|
||||||
|
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
|
||||||
|
touch ${FOLDER}/${nip}.down
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
nid=`expr $nid + 1`
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
||||||
|
|||||||
Reference in New Issue
Block a user