voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update nodemgmt-scripts.sh

Browse Source
This commit is contained in:
David Schroeder
2019-02-07 23:26:14 -06:00
parent cfa1b85454
commit 516aee0eec
1 changed files with 98 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
nodemgmt-scripts.sh
View File

@@ -17,17 +17,21 @@ source /opt/idssys/nodemgmt/defaults.inc
# ========================================================= #
STATUS() {
if [ "${1}" = "report" ]; then
STATUS()
start=`date +%s`
ST_ACTION=${1}
if [ "${ST_ACTION}" = "report" ]; then
unset idsCL idsBG idsST
idsCL=('')
idsBG=('')
idsST=('')
fi
declare -i cw; declare -i spc1; declare -i c
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then
echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}"
echo
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}"
echo
fi
if [ ! -z ${LOCAL_SERVICES+x} ];
then NHCMD='ssh root@10.5.10.51'
else NHCMD=''
@@ -43,30 +47,41 @@ STATUS() {
yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
fi
done
if [ "${1}" = "repl" ]; then
if [ "${ST_ACTION}" = "repl" ]; then
sleep 5
fi
fi
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "services" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
if [ ! -z ${LOCAL_SERVICES+x} ]; then
lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
uptime=`uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
DIVIDER false green
if [ "${ST_ACTION}" != "check" ]; then
lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
uptime=`uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
DIVIDER false green
fi
for srvc in "${LOCAL_SERVICES[@]}"; do
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(systemctl is-active ${srvc})" = "active" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${FOLDER}/localhost-${srvc}.down
else
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
touch ${FOLDER}/localhost-${srvc}.down
fi
done
@@ -77,25 +92,33 @@ STATUS() {
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}"
DIVIDER false green
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
rm -f ${FOLDER}/${nip}.down
for srvc in "${NODE_SERVICES[@]}"; do
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" = "active" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
elif [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${srvc}.down
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
@@ -104,41 +127,53 @@ STATUS() {
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
echo -e "${idsCL[Red]}Not Running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${srvc}.down
else
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
fi
fi
done
else
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}.down
fi
echo
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
fi
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${NODE_HOSTS[@]}"; do
if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
rm -f ${FOLDER}/${nip}.down
for rcheck in "${REPL_CHECK[@]}"; do
c=0; cw=22; spc=''
spc1=${cw}-${#REPL_DESC[${rcheck}]}
timeout=`date --date='10 seconds' +%s`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=22; spc=''
spc1=${cw}-${#REPL_DESC[${rcheck}]}
timeout=`date --date='10 seconds' +%s`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
fi
checked=false
until [ "${checked}" = "" ]; do
if [ "${NHCMD}" = "" ]; then
@@ -146,36 +181,45 @@ STATUS() {
else
checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
fi
#cdate=`date +%s`
if [ "`date +%s`" -gt "$timeout" ]; then
timeout=true
break
fi
done
if [ "${timeout}" != "true" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
fi
rm -f ${FOLDER}/${nip}-${rcheck}.down
else
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${rcheck}.down
#echo "Timeout occurred in waiting for replication between primary node and iDS-Node${nid} (${nip}) for the replicated folder of '${rcheck}'." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
fi
done
else
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}.down
fi
echo
if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi
nid=`expr $nid + 1`
done
fi
echo ""
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
if [ "${ST_ACTION}" != "check" ]; then
echo ""
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
fi
end=`date +%s`
runtime=$((end-start))
echo "runtime: ${runtime}"
}
STATUS-CHECK(){