voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Update nodemgmt-scripts.sh

Browse Source
This commit is contained in:
David Schroeder
2019-02-07 23:26:14 -06:00
parent cfa1b85454
commit 516aee0eec
1 changed files with 98 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
nodemgmt-scripts.sh
View File

@@ -17,17 +17,21 @@ source /opt/idssys/nodemgmt/defaults.inc
# ========================================================= # # ========================================================= #
STATUS() { STATUS()
if [ "${1}" = "report" ]; then start=`date +%s`
ST_ACTION=${1}
if [ "${ST_ACTION}" = "report" ]; then
unset idsCL idsBG idsST unset idsCL idsBG idsST
idsCL=('') idsCL=('')
idsBG=('') idsBG=('')
idsST=('') idsST=('')
fi fi
declare -i cw; declare -i spc1; declare -i c declare -i cw; declare -i spc1; declare -i c
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ]; then
echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo echo -e "${idsCL[LightCyan]}Setting up status checks...${idsCL[Default]}"
echo
fi
if [ ! -z ${LOCAL_SERVICES+x} ]; if [ ! -z ${LOCAL_SERVICES+x} ];
then NHCMD='ssh root@10.5.10.51' then NHCMD='ssh root@10.5.10.51'
else NHCMD='' else NHCMD=''
@@ -43,30 +47,41 @@ STATUS() {
yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl" yes | ${NHCMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
fi fi
done done
if [ "${1}" = "repl" ]; then if [ "${ST_ACTION}" = "repl" ]; then
sleep 5 sleep 5
fi fi
fi fi
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "services" ]; then if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}" echo -e "${idsST[Bold]}"; DIVIDER
DIVIDER; echo -e "${idsST[Reset]}" echo -e "${idsCL[Yellow]} Node Service Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
if [ ! -z ${LOCAL_SERVICES+x} ]; then if [ ! -z ${LOCAL_SERVICES+x} ]; then
lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) if [ "${ST_ACTION}" != "check" ]; then
uptime=`uptime -p` lip=$(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" uptime=`uptime -p`
DIVIDER false green echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
DIVIDER false green
fi
for srvc in "${LOCAL_SERVICES[@]}"; do for srvc in "${LOCAL_SERVICES[@]}"; do
c=0; cw=22; spc='' if [ "${ST_ACTION}" != "check" ]; then
spc1=${cw}-${#NM_SERVICES[${srvc}]} c=0; cw=22; spc=''
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done spc1=${cw}-${#NM_SERVICES[${srvc}]}
echo -en " ${NM_SERVICES[${srvc}]}$spc: " until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(systemctl is-active ${srvc})" = "active" ]; then if [ "$(systemctl is-active ${srvc})" = "active" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${FOLDER}/localhost-${srvc}.down rm -f ${FOLDER}/localhost-${srvc}.down
else else
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
touch ${FOLDER}/localhost-${srvc}.down touch ${FOLDER}/localhost-${srvc}.down
fi fi
done done
@@ -77,25 +92,33 @@ STATUS() {
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH='' else NCMD="ssh root@${nip}"; LH=''
fi fi
uptime=`${NCMD} uptime -p` if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${LH}${idsCL[Default]}" uptime=`${NCMD} uptime -p`
DIVIDER false green echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ "${NCMD}" != "" ]; then if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip}) checkhost=$(CHECK_HOST ${nip})
fi fi
if [ "${checkhost}" != "false" ]; then if [ "${checkhost}" != "false" ]; then
rm -f ${FOLDER}/${nip}.down rm -f ${FOLDER}/${nip}.down
for srvc in "${NODE_SERVICES[@]}"; do for srvc in "${NODE_SERVICES[@]}"; do
c=0; cw=22; spc='' if [ "${ST_ACTION}" != "check" ]; then
spc1=${cw}-${#NM_SERVICES[${srvc}]} c=0; cw=22; spc=''
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done spc1=${cw}-${#NM_SERVICES[${srvc}]}
echo -en " ${NM_SERVICES[${srvc}]}$spc: " until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
if [ "$(${NCMD} systemctl is-active ${srvc})" = "active" ]; then if [ "$(${NCMD} systemctl is-active ${srvc})" = "active" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${FOLDER}/${nip}-${srvc}.down rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime rm -f ${FOLDER}/${nip}-${srvc}.errtime
elif [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then elif [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${srvc}.down touch ${FOLDER}/${nip}-${srvc}.down
else else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
@@ -104,41 +127,53 @@ STATUS() {
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
echo -e "${idsCL[Red]}Not Running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${srvc}.down touch ${FOLDER}/${nip}-${srvc}.down
else else
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
fi fi
fi fi
done done
else else
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}.down touch ${FOLDER}/${nip}.down
fi fi
echo if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1` nid=`expr $nid + 1`
done done
fi fi
if [ "${1}" = "" ] || [ "${1}" = "report" ] || [ "${1}" = "repl" ]; then if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then
echo -e "${idsST[Bold]}"; DIVIDER if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" echo -e "${idsST[Bold]}"; DIVIDER
DIVIDER; echo -e "${idsST[Reset]}" echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1 nid=1
for nip in "${NODE_HOSTS[@]}"; do for nip in "${NODE_HOSTS[@]}"; do
if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi if [ "${nip}" = '10.5.10.51' ] && [ ! -z ${LOCAL_SERVICES+x} ]; then isreplhost=true; else isreplhost=false; fi
if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then if [[ $(/sbin/ip -o -4 addr list ens192 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${isreplhost}" = "false" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
DIVIDER false green echo -e " ${idsST[Bold]}${idsCL[LightCyan]}iDS-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
checkhost=$(CHECK_HOST ${nip}) checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then if [ "${checkhost}" != "false" ]; then
rm -f ${FOLDER}/${nip}.down rm -f ${FOLDER}/${nip}.down
for rcheck in "${REPL_CHECK[@]}"; do for rcheck in "${REPL_CHECK[@]}"; do
c=0; cw=22; spc='' if [ "${ST_ACTION}" != "check" ]; then
spc1=${cw}-${#REPL_DESC[${rcheck}]} c=0; cw=22; spc=''
timeout=`date --date='10 seconds' +%s` spc1=${cw}-${#REPL_DESC[${rcheck}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done timeout=`date --date='10 seconds' +%s`
echo -en " ${REPL_DESC[${rcheck}]}${spc}: " until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
fi
checked=false checked=false
until [ "${checked}" = "" ]; do until [ "${checked}" = "" ]; do
if [ "${NHCMD}" = "" ]; then if [ "${NHCMD}" = "" ]; then
@@ -146,36 +181,45 @@ STATUS() {
else else
checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" checked="`${NHCMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
fi fi
#cdate=`date +%s`
if [ "`date +%s`" -gt "$timeout" ]; then if [ "`date +%s`" -gt "$timeout" ]; then
timeout=true timeout=true
break break
fi fi
done done
if [ "${timeout}" != "true" ]; then if [ "${timeout}" != "true" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
fi
rm -f ${FOLDER}/${nip}-${rcheck}.down rm -f ${FOLDER}/${nip}-${rcheck}.down
else else
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${rcheck}.down touch ${FOLDER}/${nip}-${rcheck}.down
#echo "Timeout occurred in waiting for replication between primary node and iDS-Node${nid} (${nip}) for the replicated folder of '${rcheck}'." | mail -s "Status-Check" ${STATUS_CHECK_EMAIL}
fi fi
done done
else else
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}.down touch ${FOLDER}/${nip}.down
fi fi
echo if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi fi
nid=`expr $nid + 1` nid=`expr $nid + 1`
done done
fi fi
echo "" if [ "${ST_ACTION}" != "check" ]; then
if [ -z $action ] || [ "${action}" = "gui" ]; then echo ""
DIVIDER true if [ -z $action ] || [ "${action}" = "gui" ]; then
ENTER2CONTINUE DIVIDER true
ENTER2CONTINUE
fi
fi fi
end=`date +%s`
runtime=$((end-start))
echo "runtime: ${runtime}"
} }
STATUS-CHECK(){ STATUS-CHECK(){