voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-07-04 11:09:40 -05:00
parent f53920a042
commit 3f8e796124
6 changed files with 199 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
inc/status.inc
View File

@@ -18,13 +18,13 @@ STATUS(){
fi
for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
if [ ! -f ${FOLDER}/${!PH}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
# echo "CHECK: ${NTYPE} - ${!PH} - $rcheck - ${NM_REPL_CHECK_LOC[${rcheck}]}"
${PH_CMD} rm -f ${FOLDER}/test.repl
${PH_CMD} rm -f ${NM_FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" &
done
@@ -73,39 +73,39 @@ STATUS(){
fi
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
mv ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
fi
rm -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${LOGFILE}
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE}
fi
fi
@@ -161,13 +161,13 @@ STATUS(){
DIVIDER false green
fi
if [ -f ${TMPFOLDER}/${nip}.down ]; then
if [ -f ${TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
rm -f ${TMPFOLDER}/${nip}.*
rm -f ${NM_TMPFOLDER}/${nip}.*
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
@@ -197,21 +197,21 @@ STATUS(){
fi
# echo "mysqlgo=${mysqlgo}"
if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
touch ${TMPFOLDER}/${nip}~${srvc}.down
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
[ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
touch ${TMPFOLDER}/${nip}~${srvc}.down
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
fi
@@ -219,17 +219,17 @@ STATUS(){
fi
else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
rm -f ${TMPFOLDER}/${nip}~${srvc}.*
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
fi
# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
if [ "${srvc}" == "gitea" ]; then
rm -f ${FOLDER}/*~${srvc}.*
rm -f ${NM_FOLDER}/*~${srvc}.*
fi
fi
[ "${mysqlgo}" = "true" ] && unset mysqlgo
@@ -267,22 +267,22 @@ STATUS(){
# fi
# # echo "mysqlgo=${mysqlgo}"
# if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
# if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# touch ${TMPFOLDER}/${nip}~${srvc}.down
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
# touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
#
# else
# errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
# errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
# if [ $errtime -gt ${RENOTIFY} ]; then
# if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime
# if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
# fi
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
# touch ${TMPFOLDER}/${nip}~${srvc}.down
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
# touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# if [ "${srvc}" = "mysql" ]; then
# ${NCMD} systemctl restart ${srvc}
# fi
@@ -295,23 +295,23 @@ STATUS(){
# if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Green]}Running${idsCL[Default]}"
# fi
# if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
# if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
# else
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
#
# rm -f ${TMPFOLDER}/${nip}~${srvc}.down
# rm -f ${TMPFOLDER}/${nip}~${srvc}.errtime
# rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.down
# rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
#
# fi
# # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
# if [ "${srvc}" == "gitea" ]; then
# rm -f ${FOLDER}/*~${srvc}.down
# rm -f ${FOLDER}/*~${srvc}.errtime
# rm -f ${NM_FOLDER}/*~${srvc}.down
# rm -f ${NM_FOLDER}/*~${srvc}.errtime
# fi
# fi
# [ "${mysqlgo}" = "true" ] && unset mysqlgo
@@ -346,10 +346,10 @@ STATUS(){
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
else
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
@@ -357,28 +357,28 @@ STATUS(){
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed"
rm -f ${TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE}
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
touch ${TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${TMPFOLDER}/${nip}~${docker}.down ${TMPFOLDER}/${nip}~${docker}.errtime
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${TMPFOLDER}/${nip}~${docker}.down
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
fi
fi
@@ -388,34 +388,34 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${TMPFOLDER}/${nip}~${docker}.*
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
fi
fi
done
fi
else
if [ ! -f ${TMPFOLDER}/${nip}.down ]; then
touch ${TMPFOLDER}/${nip}.down
if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
touch ${TMPFOLDER}/${nip}.errtime
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
touch ${NM_TMPFOLDER}/${nip}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
if [ $errtime -gt ${RENOTIFY} ]; then
touch ${TMPFOLDER}/${nip}.down
touch ${NM_TMPFOLDER}/${nip}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
fi
# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then
@@ -452,7 +452,7 @@ STATUS(){
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
@@ -476,7 +476,7 @@ STATUS(){
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then
@@ -491,34 +491,34 @@ STATUS(){
checked=""
until [ "${checked}" != "" ]; do
if [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
if [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
checked=good
elif [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
elif [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
checked=timeout
fi
rm -f ${TMPFOLDER}/repl.${rcheck}.${nip}.*
rm -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.*
done
if [ "${checked}" == "timeout" ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${TMPFOLDER}/${nip}~${rcheck}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
fi
else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${TMPFOLDER}/${nip}~${rcheck}.down
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}
fi
fi
done
else
if [ "${ST_ACTION}" != "check" ]; then
[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
[ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
[ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
[ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
@@ -538,25 +538,25 @@ STATUS(){
#WHCMD="ssh root@${WEB_HOSTS[0]}"
for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
if [ ! -f ${FOLDER}/${!PH}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${FOLDER}/test.repl
${PH_CMD} rm -f ${NM_FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${PH_CMD}" = "" ]; then
rm -f ${FOLDER}/test.repl &
rm -f ${NM_FOLDER}/test.repl &
rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
else
${PH_CMD} rm -f ${FOLDER}/test.repl &
${PH_CMD} rm -f ${NM_FOLDER}/test.repl &
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
fi
done
fi
fi
done
rm -Rf ${FOLDER}/test.repl
rm -Rf ${NM_FOLDER}/test.repl
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo
@@ -709,5 +709,5 @@ REPLCHECK(){
fi
[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
done
[ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good
[ "${timeout}" == "true" ] && touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good
}