Update nodemgmt-scripts.sh
This commit is contained in:
@@ -25,6 +25,32 @@ STATUS(){
|
||||
ST_ACTION=repl
|
||||
fi
|
||||
declare -i cw; declare -i spc1; declare -i c
|
||||
|
||||
########################
|
||||
if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -en "${idsCL[LightCyan]}Setting up replication checks for next status check ... ${idsCL[Default]}"
|
||||
fi
|
||||
for NTYPE in "${NODE_TYPES[@]}"; do
|
||||
PH=${NTYPE}_HOSTS[0]
|
||||
if [ ! -f ${FOLDER}/${!PH}.down ]; then
|
||||
PH_CMD="ssh root@${!PH}"
|
||||
var=${NTYPE}_REPL_CHECK[@]
|
||||
if [ ! -z ${!var+x} ]; then
|
||||
for rcheck in "${!var}"; do
|
||||
# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
|
||||
${PH_CMD} rm -f ${FOLDER}/test.repl
|
||||
daterun=`date +%Y-%m-%d-%H-%M-%S`
|
||||
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" &
|
||||
done
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
||||
echo
|
||||
fi
|
||||
fi
|
||||
|
||||
########################
|
||||
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then
|
||||
@@ -340,10 +366,113 @@ STATUS(){
|
||||
########################
|
||||
|
||||
if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
|
||||
if [ ! -f ${FOLDER}/test.repl ]; then
|
||||
touch ${FOLDER}/test.repl
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
|
||||
DIVIDER; echo -e "${idsST[Reset]}"
|
||||
fi
|
||||
|
||||
for NTYPE in "${NODE_TYPES[@]}"; do
|
||||
PH=${NTYPE}_HOSTS[0]
|
||||
PH_CMD="ssh root@${!PH}"
|
||||
var=${NTYPE}_REPL_CHECK[@]
|
||||
if [ ! -z ${!var+x} ]; then
|
||||
nid=1
|
||||
|
||||
var=${NTYPE}_HOSTS[@]
|
||||
for nip in "${!var}"; do
|
||||
|
||||
# echo "CHECK: ${NTYPE} - ${nip}"
|
||||
|
||||
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
fi
|
||||
|
||||
# checkhost=$(CHECK_HOST ${nip})
|
||||
#if [ "${checkhost}" != "false" ]; then
|
||||
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
|
||||
var2=${NTYPE}_REPL_CHECK[@]
|
||||
# timeout=`date --date='20 seconds' +%s`
|
||||
for rcheck in "${!var2}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${ST_ACTION}" != "report" ]; then
|
||||
c=0; cw=30; spc=''
|
||||
spc1=${cw}-${#REPL_DESC[${rcheck}]}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
else
|
||||
spc=' '
|
||||
fi
|
||||
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
|
||||
|
||||
timeout=`date --date='2 minutes' +%s`
|
||||
|
||||
# if [ "${rcheck}" == "www" ]; then
|
||||
# timeout=`date --date='2 minutes' +%s`
|
||||
# else
|
||||
# timeout=`date --date='20 seconds' +%s`
|
||||
# fi
|
||||
|
||||
else
|
||||
timeout=`date --date='2 minutes' +%s`
|
||||
fi
|
||||
warn_timeout=`date --date='30 secs' +%s`
|
||||
|
||||
checked=false
|
||||
until [ "${checked}" = "" ]; do
|
||||
if [ "${PH_CMD}" = "" ]; then
|
||||
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
|
||||
else
|
||||
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
|
||||
fi
|
||||
if [ "`date +%s`" -gt "${timeout}" ]; then
|
||||
timeout=true
|
||||
break
|
||||
elif [ "`date +%s`" -gt "${warn_timeout}" ]; then
|
||||
echo -en " waiting for sync... "
|
||||
warn_timeout=`date --date='100 minutes' +%s`
|
||||
fi
|
||||
done
|
||||
if [ "${timeout}" = "true" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
||||
fi
|
||||
if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
|
||||
touch ${FOLDER}/${nip}-${rcheck}.down
|
||||
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
|
||||
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
|
||||
fi
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
|
||||
fi
|
||||
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
|
||||
rm -f ${FOLDER}/${nip}-${rcheck}.down
|
||||
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
|
||||
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
|
||||
fi
|
||||
fi
|
||||
|
||||
done
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}"
|
||||
[ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}"
|
||||
fi
|
||||
fi
|
||||
if [ "${ST_ACTION}" != "check" ]; then echo; fi
|
||||
fi
|
||||
nid=`expr $nid + 1`
|
||||
done
|
||||
fi
|
||||
done
|
||||
##########################
|
||||
# REMOVE REPL CHECK FILES
|
||||
##########################
|
||||
if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -en "${idsCL[LightCyan]}Setting up replication checks for next status check ... ${idsCL[Default]}"
|
||||
echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
|
||||
fi
|
||||
#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
|
||||
#LBHCMD="ssh root@${LB_HOSTS[0]}"
|
||||
@@ -355,187 +484,25 @@ STATUS(){
|
||||
var=${NTYPE}_REPL_CHECK[@]
|
||||
if [ ! -z ${!var+x} ]; then
|
||||
for rcheck in "${!var}"; do
|
||||
# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
|
||||
${PH_CMD} rm -f ${FOLDER}/test.repl
|
||||
daterun=`date +%Y-%m-%d-%H-%M-%S`
|
||||
if [ "${PH_CMD}" = "" ]; then
|
||||
echo -e "${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
|
||||
yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
|
||||
rm -f ${FOLDER}/test.repl &
|
||||
rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
|
||||
else
|
||||
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl"
|
||||
# ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
|
||||
# yes | ${PH_CMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
|
||||
${PH_CMD} rm -f ${FOLDER}/test.repl &
|
||||
${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
done
|
||||
rm -Rf ${FOLDER}/test.repl
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
||||
echo
|
||||
fi
|
||||
[ "${ST_ACTION}" != "repl" ] && CHECKFOLDERSYNC=false || CHECKFOLDERSYNC=true
|
||||
else
|
||||
rm -Rf ${FOLDER}/test.repl
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo
|
||||
echo -e "${idsCL[LightCyan]}Will verify folder replication this status check ... ${idsCL[Default]}"
|
||||
CHECKFOLDERSYNC=true
|
||||
fi
|
||||
fi
|
||||
|
||||
########################
|
||||
|
||||
if [ "${CHECKFOLDERSYNC}" == true ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
|
||||
DIVIDER; echo -e "${idsST[Reset]}"
|
||||
fi
|
||||
|
||||
|
||||
for NTYPE in "${NODE_TYPES[@]}"; do
|
||||
#GOREPCHK=false;
|
||||
#CHR=`date +"%H"`;
|
||||
#if [ "${CHR}" -ge "12" ] && [ "${CHR}" -le "13" ] && [ "${NTYPE}" = "NC" ]; then
|
||||
# GOREPCHK=true;
|
||||
#elif [ "${NTYPE}" != "NC" ]; then
|
||||
# GOREPCHK=true;
|
||||
#fi
|
||||
GOREPCHK=true;
|
||||
if [ "${GOREPCHK}" = "true" ]; then
|
||||
PH=${NTYPE}_HOSTS[0]
|
||||
PH_CMD="ssh root@${!PH}"
|
||||
var=${NTYPE}_REPL_CHECK[@]
|
||||
if [ ! -z ${!var+x} ]; then
|
||||
nid=1
|
||||
|
||||
var=${NTYPE}_HOSTS[@]
|
||||
for nip in "${!var}"; do
|
||||
|
||||
# echo "CHECK: ${NTYPE} - ${nip}"
|
||||
|
||||
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
fi
|
||||
|
||||
# checkhost=$(CHECK_HOST ${nip})
|
||||
#if [ "${checkhost}" != "false" ]; then
|
||||
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
|
||||
var2=${NTYPE}_REPL_CHECK[@]
|
||||
# timeout=`date --date='20 seconds' +%s`
|
||||
for rcheck in "${!var2}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${ST_ACTION}" != "report" ]; then
|
||||
c=0; cw=30; spc=''
|
||||
spc1=${cw}-${#REPL_DESC[${rcheck}]}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
else
|
||||
spc=' '
|
||||
fi
|
||||
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
|
||||
|
||||
timeout=`date --date='2 minutes' +%s`
|
||||
|
||||
# if [ "${rcheck}" == "www" ]; then
|
||||
# timeout=`date --date='2 minutes' +%s`
|
||||
# else
|
||||
# timeout=`date --date='20 seconds' +%s`
|
||||
# fi
|
||||
|
||||
else
|
||||
timeout=`date --date='2 minutes' +%s`
|
||||
fi
|
||||
warn_timeout=`date --date='30 secs' +%s`
|
||||
|
||||
checked=false
|
||||
until [ "${checked}" = "" ]; do
|
||||
if [ "${PH_CMD}" = "" ]; then
|
||||
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
|
||||
else
|
||||
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
|
||||
fi
|
||||
if [ "`date +%s`" -gt "${timeout}" ]; then
|
||||
timeout=true
|
||||
break
|
||||
elif [ "`date +%s`" -gt "${warn_timeout}" ]; then
|
||||
echo -en " waiting for sync... "
|
||||
warn_timeout=`date --date='100 minutes' +%s`
|
||||
fi
|
||||
done
|
||||
if [ "${timeout}" = "true" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
||||
fi
|
||||
if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
|
||||
touch ${FOLDER}/${nip}-${rcheck}.down
|
||||
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
|
||||
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
|
||||
fi
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
|
||||
fi
|
||||
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
|
||||
rm -f ${FOLDER}/${nip}-${rcheck}.down
|
||||
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
|
||||
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
|
||||
fi
|
||||
fi
|
||||
|
||||
done
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) is offline${idsCL[Default]}"
|
||||
[ -f ${FOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_NODETYPES[$NTYPE]}-Node${nip: -1} (${nip}) is offline${idsCL[Default]}"
|
||||
fi
|
||||
fi
|
||||
if [ "${ST_ACTION}" != "check" ]; then echo; fi
|
||||
fi
|
||||
nid=`expr $nid + 1`
|
||||
done
|
||||
fi
|
||||
fi
|
||||
done
|
||||
##########################
|
||||
# REMOVE REPL CHECK FILES
|
||||
##########################
|
||||
if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
|
||||
fi
|
||||
#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
|
||||
#LBHCMD="ssh root@${LB_HOSTS[0]}"
|
||||
#WHCMD="ssh root@${WEB_HOSTS[0]}"
|
||||
for NTYPE in "${NODE_TYPES[@]}"; do
|
||||
PH=${NTYPE}_HOSTS[0]
|
||||
if [ ! -f ${FOLDER}/${!PH}.down ]; then
|
||||
PH_CMD="ssh root@${!PH}"
|
||||
var=${NTYPE}_REPL_CHECK[@]
|
||||
if [ ! -z ${!var+x} ]; then
|
||||
for rcheck in "${!var}"; do
|
||||
${PH_CMD} rm -f ${FOLDER}/test.repl
|
||||
daterun=`date +%Y-%m-%d-%H-%M-%S`
|
||||
if [ "${PH_CMD}" = "" ]; then
|
||||
rm -f ${FOLDER}/test.repl
|
||||
rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
|
||||
else
|
||||
${PH_CMD} rm -f ${FOLDER}/test.repl
|
||||
${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
done
|
||||
rm -Rf ${FOLDER}/test.repl
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
||||
echo
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
########################
|
||||
|
||||
Reference in New Issue
Block a user