Update nodemgmt-scripts.sh
This commit is contained in:
@@ -68,7 +68,7 @@ STATUS(){
|
||||
fi
|
||||
fi
|
||||
########################
|
||||
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then
|
||||
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then
|
||||
if [ ! -z ${LOCAL_SERVICES+x} ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
|
||||
@@ -133,7 +133,11 @@ STATUS(){
|
||||
for NTYPE in "${NODE_TYPES[@]}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
|
||||
if [ "${ST_ACTION}" = "dockers" ]; then
|
||||
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
|
||||
else
|
||||
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
|
||||
fi
|
||||
DIVIDER; echo -e "${idsST[Reset]}"
|
||||
fi
|
||||
nid=1
|
||||
@@ -142,194 +146,208 @@ STATUS(){
|
||||
hosts=${NTYPE}_HOSTS[@]
|
||||
|
||||
var=${NTYPE}_HOSTS[@]
|
||||
for nip in "${!var}"; do
|
||||
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
||||
else NCMD="ssh root@${nip}"; LH=''
|
||||
fi
|
||||
if [ "${NCMD}" != "" ]; then
|
||||
checkhost=$(CHECK_HOST ${nip})
|
||||
fi
|
||||
if [ "${checkhost}" != "false" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
uptime=`${NCMD} uptime -p`
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
|
||||
if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" = "dockers" ]; then
|
||||
GOFORCHECK=false;
|
||||
else
|
||||
GOFORCHECK=true;
|
||||
fi
|
||||
|
||||
if [ ${GOFORCHECK} = true ]; then
|
||||
for nip in "${!var}"; do
|
||||
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
||||
else NCMD="ssh root@${nip}"; LH=''
|
||||
fi
|
||||
if [ -f ${FOLDER}/${nip}.down ]; then
|
||||
if [ -f ${FOLDER}/${nip}.errtime ]; then
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
|
||||
else
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
|
||||
fi
|
||||
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
|
||||
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
|
||||
if [ "${NCMD}" != "" ]; then
|
||||
checkhost=$(CHECK_HOST ${nip})
|
||||
fi
|
||||
|
||||
var2=${NTYPE}_SERVICES_CHECK[@]
|
||||
for srvc in "${!var2}"; do
|
||||
NOGOCHK=true;
|
||||
if [ ${srvc} = "gitea" ] || [ ${srvc} = "pdnsadmin" ] || [ ${srvc} = "pdnsadmin.socket" ]; then
|
||||
if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.10.1.120* ]]; then
|
||||
NOGOCHK=false;
|
||||
fi
|
||||
fi
|
||||
if [ ${NOGOCHK} = true ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${ST_ACTION}" != "report" ]; then
|
||||
c=0; cw=30; spc=''
|
||||
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
else
|
||||
spc=' '
|
||||
fi
|
||||
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
||||
fi
|
||||
srvctst=$(${NCMD} systemctl is-active ${srvc})
|
||||
#echo "${NCMD} systemctl is-active ${srvc} = ${srvctst}"
|
||||
#if [ "${srvctst}" = "active" ]; then echo "yes"; else echo "no"; fi
|
||||
if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then
|
||||
mysqlgo=false;
|
||||
[ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true;
|
||||
elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
|
||||
mysqlgo=false;
|
||||
[ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true;
|
||||
elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then
|
||||
mysqlgo=true
|
||||
fi
|
||||
#echo "mysqlgo=${mysqlgo}"
|
||||
if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
|
||||
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
|
||||
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||
fi
|
||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
touch ${FOLDER}/${nip}-${srvc}.down
|
||||
# fi
|
||||
else
|
||||
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
|
||||
if [ $errtime -gt ${RENOTIFY} ]; then
|
||||
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
|
||||
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
|
||||
fi
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
||||
fi
|
||||
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
touch ${FOLDER}/${nip}-${srvc}.down
|
||||
if [ "${srvc}" = "mysql" ]; then
|
||||
${NCMD} systemctl restart ${srvc}
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||
fi
|
||||
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
|
||||
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
|
||||
else
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
|
||||
fi
|
||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
|
||||
if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
|
||||
rm -f ${FOLDER}/*-${srvc}.down
|
||||
rm -f ${FOLDER}/*-${srvc}.errtime
|
||||
else
|
||||
rm -f ${FOLDER}/${nip}-${srvc}.down
|
||||
rm -f ${FOLDER}/${nip}-${srvc}.errtime
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -v ${NTYPE}_DOCKER ]]; then
|
||||
if [ "${checkhost}" != "false" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo
|
||||
echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
|
||||
echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}"
|
||||
uptime=`${NCMD} uptime -p`
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
fi
|
||||
if [ -f ${FOLDER}/${nip}.down ]; then
|
||||
if [ -f ${FOLDER}/${nip}.errtime ]; then
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
|
||||
else
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
|
||||
fi
|
||||
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
|
||||
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
|
||||
fi
|
||||
|
||||
for docker in "${!dockers}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
c=0; cw=30; spc=''
|
||||
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
|
||||
fi
|
||||
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||
# if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
|
||||
# ${NCMD} docker rm ${docker} >/dev/null 2>&1
|
||||
# fi
|
||||
# ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
|
||||
# sleep 10s
|
||||
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
|
||||
if [ "${ST_ACTION}" != "dockers" ]; then
|
||||
var2=${NTYPE}_SERVICES_CHECK[@]
|
||||
for srvc in "${!var2}"; do
|
||||
NOGOCHK=true;
|
||||
if [ ${srvc} = "gitea" ] || [ ${srvc} = "pdnsadmin" ] || [ ${srvc} = "pdnsadmin.socket" ]; then
|
||||
if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.10.1.120* ]]; then
|
||||
NOGOCHK=false;
|
||||
fi
|
||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
rm -f ${FOLDER}/${nip}-${docker}.down
|
||||
rm -f ${FOLDER}/${nip}-${docker}.errtime
|
||||
else
|
||||
fi
|
||||
if [ ${NOGOCHK} = true ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
|
||||
|
||||
if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
|
||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
touch ${FOLDER}/${nip}-${docker}.down
|
||||
if [ "${ST_ACTION}" != "report" ]; then
|
||||
c=0; cw=30; spc=''
|
||||
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
else
|
||||
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down)
|
||||
if [ $errtime -gt ${RENOTIFY} ]; then
|
||||
if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then
|
||||
mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime
|
||||
spc=' '
|
||||
fi
|
||||
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
||||
fi
|
||||
srvctst=$(${NCMD} systemctl is-active ${srvc})
|
||||
#echo "${NCMD} systemctl is-active ${srvc} = ${srvctst}"
|
||||
#if [ "${srvctst}" = "active" ]; then echo "yes"; else echo "no"; fi
|
||||
if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then
|
||||
mysqlgo=false;
|
||||
[ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true;
|
||||
elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
|
||||
mysqlgo=false;
|
||||
[ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true;
|
||||
elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then
|
||||
mysqlgo=true
|
||||
fi
|
||||
#echo "mysqlgo=${mysqlgo}"
|
||||
if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
|
||||
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
|
||||
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||
fi
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime)
|
||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
touch ${FOLDER}/${nip}-${srvc}.down
|
||||
# fi
|
||||
else
|
||||
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
|
||||
if [ $errtime -gt ${RENOTIFY} ]; then
|
||||
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
|
||||
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
|
||||
fi
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
||||
fi
|
||||
touch ${FOLDER}/${nip}-${docker}.down
|
||||
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
touch ${FOLDER}/${nip}-${srvc}.down
|
||||
if [ "${srvc}" = "mysql" ]; then
|
||||
${NCMD} systemctl restart ${srvc}
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||
fi
|
||||
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
|
||||
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
|
||||
else
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
|
||||
fi
|
||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
|
||||
if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
|
||||
rm -f ${FOLDER}/*-${srvc}.down
|
||||
rm -f ${FOLDER}/*-${srvc}.errtime
|
||||
else
|
||||
rm -f ${FOLDER}/${nip}-${srvc}.down
|
||||
rm -f ${FOLDER}/${nip}-${srvc}.errtime
|
||||
fi
|
||||
fi
|
||||
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||
fi
|
||||
rm -f ${FOLDER}/${nip}-${docker}.down
|
||||
rm -f ${FOLDER}/${nip}-${docker}.errtime
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
else
|
||||
if [ ! -f ${FOLDER}/${nip}.down ]; then
|
||||
touch ${FOLDER}/${nip}.down
|
||||
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
|
||||
touch ${FOLDER}/${nip}.errtime
|
||||
done
|
||||
fi
|
||||
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
|
||||
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
|
||||
# fi
|
||||
|
||||
if [[ -v ${NTYPE}_DOCKER ]]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo
|
||||
echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
|
||||
echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}"
|
||||
fi
|
||||
|
||||
for docker in "${!dockers}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
c=0; cw=30; spc=''
|
||||
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
|
||||
fi
|
||||
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||
# if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
|
||||
# ${NCMD} docker rm ${docker} >/dev/null 2>&1
|
||||
# fi
|
||||
# ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
|
||||
# sleep 10s
|
||||
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
|
||||
fi
|
||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
rm -f ${FOLDER}/${nip}-${docker}.down
|
||||
rm -f ${FOLDER}/${nip}-${docker}.errtime
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
|
||||
|
||||
if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
|
||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||
touch ${FOLDER}/${nip}-${docker}.down
|
||||
else
|
||||
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down)
|
||||
if [ $errtime -gt ${RENOTIFY} ]; then
|
||||
if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then
|
||||
mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime
|
||||
fi
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime)
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
||||
fi
|
||||
touch ${FOLDER}/${nip}-${docker}.down
|
||||
fi
|
||||
fi
|
||||
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||
fi
|
||||
rm -f ${FOLDER}/${nip}-${docker}.down
|
||||
rm -f ${FOLDER}/${nip}-${docker}.errtime
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
else
|
||||
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
|
||||
# if [ $errtime -gt ${RENOTIFY} ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then
|
||||
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
|
||||
# fi
|
||||
if [ ! -f ${FOLDER}/${nip}.down ]; then
|
||||
touch ${FOLDER}/${nip}.down
|
||||
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
|
||||
touch ${FOLDER}/${nip}.errtime
|
||||
fi
|
||||
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
|
||||
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
|
||||
# fi
|
||||
else
|
||||
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
|
||||
# if [ $errtime -gt ${RENOTIFY} ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then
|
||||
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
|
||||
# fi
|
||||
fi
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
||||
fi
|
||||
fi
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
||||
fi
|
||||
fi
|
||||
if [ "${ST_ACTION}" != "check" ]; then echo; fi
|
||||
nid=`expr $nid + 1`
|
||||
done
|
||||
if [ "${ST_ACTION}" != "check" ]; then echo; fi
|
||||
nid=`expr $nid + 1`
|
||||
done
|
||||
|
||||
else #GOFORCHECK
|
||||
|
||||
fi #GOFORCHECK
|
||||
|
||||
done
|
||||
|
||||
|
||||
Reference in New Issue
Block a user