Update nodemgmt-scripts.sh

This commit is contained in:
2023-04-11 21:44:38 -05:00
parent 0b7a4bc845
commit f85f67f827

View File

@@ -27,37 +27,45 @@ STATUS(){
declare -i cw; declare -i spc1; declare -i c declare -i cw; declare -i spc1; declare -i c
######################## ########################
if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ ! -f ${FOLDER}/test.repl ]; then
echo -en "${idsCL[LightCyan]}Setting up status checks... ${idsCL[Default]}" if [ "${ST_ACTION}" != "check" ]; then
fi echo -en "${idsCL[LightCyan]}Setting up status checks... ${idsCL[Default]}"
#NHCMD="ssh root@${MYSQL_HOSTS[0]}" fi
#LBHCMD="ssh root@${LB_HOSTS[0]}" #NHCMD="ssh root@${MYSQL_HOSTS[0]}"
#WHCMD="ssh root@${WEB_HOSTS[0]}" #LBHCMD="ssh root@${LB_HOSTS[0]}"
for NTYPE in "${NODE_TYPES[@]}"; do #WHCMD="ssh root@${WEB_HOSTS[0]}"
PH=${NTYPE}_HOSTS[0] for NTYPE in "${NODE_TYPES[@]}"; do
PH_CMD="ssh root@${!PH}" PH=${NTYPE}_HOSTS[0]
var=${NTYPE}_REPL_CHECK[@] PH_CMD="ssh root@${!PH}"
if [ ! -z ${!var+x} ]; then var=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var}"; do if [ ! -z ${!var+x} ]; then
# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}" for rcheck in "${!var}"; do
${PH_CMD} rm -f ${FOLDER}/test.repl # echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
daterun=`date +%Y-%m-%d-%H-%M-%S` ${PH_CMD} rm -f ${FOLDER}/test.repl
if [ "${PH_CMD}" = "" ]; then daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl if [ "${PH_CMD}" = "" ]; then
yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl echo -e "${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
else yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl" else
yes | ${PH_CMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl" ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
fi yes | ${PH_CMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
done fi
done
fi
done
if [ "${ST_ACTION}" = "repl" ]; then
sleep 5
fi
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo
fi
CHECKFOLDERSYNC=false;
else
if [ "${ST_ACTION}" != "check" ]; then
echo -en "${idsCL[LightCyan]}Will verify folder syncs... ${idsCL[Default]}"
CHECKFOLDERSYNC=true;
fi fi
done
if [ "${ST_ACTION}" = "repl" ]; then
sleep 5
fi
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo
fi fi
fi fi
######################## ########################
@@ -371,164 +379,167 @@ STATUS(){
## REPLICATION CHECK ## REPLICATION CHECK
######################## ########################
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${CHECKFOLDERSYNC}" == true ]; then
echo -e "${idsST[Bold]}"; DIVIDER if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" echo -e "${idsST[Bold]}"; DIVIDER
DIVIDER; echo -e "${idsST[Reset]}" echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
for NTYPE in "${NODE_TYPES[@]}"; do
#GOREPCHK=false;
#CHR=`date +"%H"`;
#if [ "${CHR}" -ge "12" ] && [ "${CHR}" -le "13" ] && [ "${NTYPE}" = "NC" ]; then
# GOREPCHK=true;
#elif [ "${NTYPE}" != "NC" ]; then
# GOREPCHK=true;
#fi
GOREPCHK=true;
if [ "${GOREPCHK}" = "true" ]; then
PH=${NTYPE}_HOSTS[0]
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
nid=1
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
# echo "CHECK: ${NTYPE} - ${nip}"
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down
rm -f ${FOLDER}/${nip}.errtime
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
var2=${NTYPE}_REPL_CHECK[@]
# timeout=`date --date='20 seconds' +%s`
for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''
spc1=${cw}-${#REPL_DESC[${rcheck}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
# if [ "${rcheck}" = "nextcloud-data" ] || [ "${rcheck}" = "nextcloud-www" ]; then
# timeout=`date --date='2 minutes' +%s`
# else
timeout=`date --date='20 seconds' +%s`
# fi
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
else
# if [ "${rcheck}" = "nextcloud-data" ] OR [ "${rcheck}" = "nextcloud-www" ]; then
# timeout=`date --date='2 minutes' +%s`
# else
timeout=`date --date='1 minutes' +%s`
# fi
fi
checked=false
until [ "${checked}" = "" ]; do
if [ "${PH_CMD}" = "" ]; then
checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
else
checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
fi
if [ "`date +%s`" -gt "$timeout" ]; then
timeout=true
break
fi
done
if [ "${timeout}" = "true" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
fi
if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
rm -f ${FOLDER}/${nip}-${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi
fi
done
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
if [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi
nid=`expr $nid + 1`
done
fi
fi
done
fi fi
##########################
# REMOVE REPL CHECK FILES
for NTYPE in "${NODE_TYPES[@]}"; do ##########################
#GOREPCHK=false; if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
#CHR=`date +"%H"`; if [ "${ST_ACTION}" != "check" ]; then
#if [ "${CHR}" -ge "12" ] && [ "${CHR}" -le "13" ] && [ "${NTYPE}" = "NC" ]; then echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
# GOREPCHK=true; fi
#elif [ "${NTYPE}" != "NC" ]; then #NHCMD="ssh root@${MYSQL_HOSTS[0]}"
# GOREPCHK=true; #LBHCMD="ssh root@${LB_HOSTS[0]}"
#fi #WHCMD="ssh root@${WEB_HOSTS[0]}"
GOREPCHK=true; for NTYPE in "${NODE_TYPES[@]}"; do
if [ "${GOREPCHK}" = "true" ]; then
PH=${NTYPE}_HOSTS[0] PH=${NTYPE}_HOSTS[0]
PH_CMD="ssh root@${!PH}" PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@] var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then if [ ! -z ${!var+x} ]; then
nid=1 for rcheck in "${!var}"; do
${PH_CMD} rm -f ${FOLDER}/test.repl
var=${NTYPE}_HOSTS[@] daterun=`date +%Y-%m-%d-%H-%M-%S`
for nip in "${!var}"; do if [ "${PH_CMD}" = "" ]; then
rm -f ${FOLDER}/test.repl
# echo "CHECK: ${NTYPE} - ${nip}" rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
else
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then ${PH_CMD} rm -f ${FOLDER}/test.repl
if [ "${ST_ACTION}" != "check" ]; then ${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down
rm -f ${FOLDER}/${nip}.errtime
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
var2=${NTYPE}_REPL_CHECK[@]
# timeout=`date --date='20 seconds' +%s`
for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''
spc1=${cw}-${#REPL_DESC[${rcheck}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
# if [ "${rcheck}" = "nextcloud-data" ] || [ "${rcheck}" = "nextcloud-www" ]; then
# timeout=`date --date='2 minutes' +%s`
# else
timeout=`date --date='20 seconds' +%s`
# fi
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
else
# if [ "${rcheck}" = "nextcloud-data" ] OR [ "${rcheck}" = "nextcloud-www" ]; then
# timeout=`date --date='2 minutes' +%s`
# else
timeout=`date --date='1 minutes' +%s`
# fi
fi
checked=false
until [ "${checked}" = "" ]; do
if [ "${PH_CMD}" = "" ]; then
checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
else
checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
fi
if [ "`date +%s`" -gt "$timeout" ]; then
timeout=true
break
fi
done
if [ "${timeout}" = "true" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
fi
if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
rm -f ${FOLDER}/${nip}-${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi
fi
done
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
if [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi fi
nid=`expr $nid + 1`
done done
fi fi
done
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo
fi fi
done fi
fi
##########################
# REMOVE REPL CHECK FILES
##########################
if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
fi
#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
#LBHCMD="ssh root@${LB_HOSTS[0]}"
#WHCMD="ssh root@${WEB_HOSTS[0]}"
for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${PH_CMD}" = "" ]; then
rm -f ${FOLDER}/test.repl
rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
else
${PH_CMD} rm -f ${FOLDER}/test.repl
${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
fi
done
fi
done
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo
fi
fi fi
######################## ########################
## FREE SPACE CHECK ## FREE SPACE CHECK
######################## ########################