Update nodemgmt-scripts.sh
This commit is contained in:
@@ -68,7 +68,7 @@ STATUS(){
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
########################
|
########################
|
||||||
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then
|
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then
|
||||||
if [ ! -z ${LOCAL_SERVICES+x} ]; then
|
if [ ! -z ${LOCAL_SERVICES+x} ]; then
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
|
lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
|
||||||
@@ -133,7 +133,11 @@ STATUS(){
|
|||||||
for NTYPE in "${NODE_TYPES[@]}"; do
|
for NTYPE in "${NODE_TYPES[@]}"; do
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
echo -e "${idsST[Bold]}"; DIVIDER
|
echo -e "${idsST[Bold]}"; DIVIDER
|
||||||
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
|
if [ "${ST_ACTION}" = "dockers" ]; then
|
||||||
|
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
|
||||||
|
else
|
||||||
|
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
|
||||||
|
fi
|
||||||
DIVIDER; echo -e "${idsST[Reset]}"
|
DIVIDER; echo -e "${idsST[Reset]}"
|
||||||
fi
|
fi
|
||||||
nid=1
|
nid=1
|
||||||
@@ -142,194 +146,208 @@ STATUS(){
|
|||||||
hosts=${NTYPE}_HOSTS[@]
|
hosts=${NTYPE}_HOSTS[@]
|
||||||
|
|
||||||
var=${NTYPE}_HOSTS[@]
|
var=${NTYPE}_HOSTS[@]
|
||||||
for nip in "${!var}"; do
|
|
||||||
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" = "dockers" ]; then
|
||||||
else NCMD="ssh root@${nip}"; LH=''
|
GOFORCHECK=false;
|
||||||
fi
|
else
|
||||||
if [ "${NCMD}" != "" ]; then
|
GOFORCHECK=true;
|
||||||
checkhost=$(CHECK_HOST ${nip})
|
fi
|
||||||
fi
|
|
||||||
if [ "${checkhost}" != "false" ]; then
|
if [ ${GOFORCHECK} = true ]; then
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
for nip in "${!var}"; do
|
||||||
uptime=`${NCMD} uptime -p`
|
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
||||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
else NCMD="ssh root@${nip}"; LH=''
|
||||||
DIVIDER false green
|
|
||||||
fi
|
fi
|
||||||
if [ -f ${FOLDER}/${nip}.down ]; then
|
if [ "${NCMD}" != "" ]; then
|
||||||
if [ -f ${FOLDER}/${nip}.errtime ]; then
|
checkhost=$(CHECK_HOST ${nip})
|
||||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
|
|
||||||
else
|
|
||||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
|
|
||||||
fi
|
|
||||||
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
|
|
||||||
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
|
|
||||||
fi
|
fi
|
||||||
|
if [ "${checkhost}" != "false" ]; then
|
||||||
var2=${NTYPE}_SERVICES_CHECK[@]
|
|
||||||
for srvc in "${!var2}"; do
|
|
||||||
NOGOCHK=true;
|
|
||||||
if [ ${srvc} = "gitea" ] || [ ${srvc} = "pdnsadmin" ] || [ ${srvc} = "pdnsadmin.socket" ]; then
|
|
||||||
if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.10.1.120* ]]; then
|
|
||||||
NOGOCHK=false;
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
if [ ${NOGOCHK} = true ]; then
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
if [ "${ST_ACTION}" != "report" ]; then
|
|
||||||
c=0; cw=30; spc=''
|
|
||||||
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
|
||||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
||||||
else
|
|
||||||
spc=' '
|
|
||||||
fi
|
|
||||||
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
|
||||||
fi
|
|
||||||
srvctst=$(${NCMD} systemctl is-active ${srvc})
|
|
||||||
#echo "${NCMD} systemctl is-active ${srvc} = ${srvctst}"
|
|
||||||
#if [ "${srvctst}" = "active" ]; then echo "yes"; else echo "no"; fi
|
|
||||||
if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then
|
|
||||||
mysqlgo=false;
|
|
||||||
[ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true;
|
|
||||||
elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
|
|
||||||
mysqlgo=false;
|
|
||||||
[ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true;
|
|
||||||
elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then
|
|
||||||
mysqlgo=true
|
|
||||||
fi
|
|
||||||
#echo "mysqlgo=${mysqlgo}"
|
|
||||||
if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
|
|
||||||
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
|
|
||||||
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
|
||||||
touch ${FOLDER}/${nip}-${srvc}.down
|
|
||||||
# fi
|
|
||||||
else
|
|
||||||
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
|
|
||||||
if [ $errtime -gt ${RENOTIFY} ]; then
|
|
||||||
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
|
|
||||||
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
|
|
||||||
fi
|
|
||||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
|
||||||
touch ${FOLDER}/${nip}-${srvc}.down
|
|
||||||
if [ "${srvc}" = "mysql" ]; then
|
|
||||||
${NCMD} systemctl restart ${srvc}
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
|
|
||||||
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
|
|
||||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
|
|
||||||
else
|
|
||||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
|
|
||||||
fi
|
|
||||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
|
||||||
|
|
||||||
if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
|
|
||||||
rm -f ${FOLDER}/*-${srvc}.down
|
|
||||||
rm -f ${FOLDER}/*-${srvc}.errtime
|
|
||||||
else
|
|
||||||
rm -f ${FOLDER}/${nip}-${srvc}.down
|
|
||||||
rm -f ${FOLDER}/${nip}-${srvc}.errtime
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ -v ${NTYPE}_DOCKER ]]; then
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
echo
|
uptime=`${NCMD} uptime -p`
|
||||||
echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
|
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
||||||
echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}"
|
DIVIDER false green
|
||||||
|
fi
|
||||||
|
if [ -f ${FOLDER}/${nip}.down ]; then
|
||||||
|
if [ -f ${FOLDER}/${nip}.errtime ]; then
|
||||||
|
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
|
||||||
|
else
|
||||||
|
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
|
||||||
|
fi
|
||||||
|
rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime
|
||||||
|
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
for docker in "${!dockers}"; do
|
if [ "${ST_ACTION}" != "dockers" ]; then
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
var2=${NTYPE}_SERVICES_CHECK[@]
|
||||||
c=0; cw=30; spc=''
|
for srvc in "${!var2}"; do
|
||||||
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
|
NOGOCHK=true;
|
||||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
if [ ${srvc} = "gitea" ] || [ ${srvc} = "pdnsadmin" ] || [ ${srvc} = "pdnsadmin.socket" ]; then
|
||||||
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
|
if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.10.1.120* ]]; then
|
||||||
fi
|
NOGOCHK=false;
|
||||||
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
|
||||||
# if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
|
|
||||||
# ${NCMD} docker rm ${docker} >/dev/null 2>&1
|
|
||||||
# fi
|
|
||||||
# ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
|
|
||||||
# sleep 10s
|
|
||||||
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
|
|
||||||
fi
|
fi
|
||||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
fi
|
||||||
rm -f ${FOLDER}/${nip}-${docker}.down
|
if [ ${NOGOCHK} = true ]; then
|
||||||
rm -f ${FOLDER}/${nip}-${docker}.errtime
|
|
||||||
else
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
|
if [ "${ST_ACTION}" != "report" ]; then
|
||||||
|
c=0; cw=30; spc=''
|
||||||
if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
|
spc1=${cw}-${#NM_SERVICES[${srvc}]}
|
||||||
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
touch ${FOLDER}/${nip}-${docker}.down
|
|
||||||
else
|
else
|
||||||
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down)
|
spc=' '
|
||||||
if [ $errtime -gt ${RENOTIFY} ]; then
|
fi
|
||||||
if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then
|
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
||||||
mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime
|
fi
|
||||||
|
srvctst=$(${NCMD} systemctl is-active ${srvc})
|
||||||
|
#echo "${NCMD} systemctl is-active ${srvc} = ${srvctst}"
|
||||||
|
#if [ "${srvctst}" = "active" ]; then echo "yes"; else echo "no"; fi
|
||||||
|
if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then
|
||||||
|
mysqlgo=false;
|
||||||
|
[ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true;
|
||||||
|
elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
|
||||||
|
mysqlgo=false;
|
||||||
|
[ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true;
|
||||||
|
elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then
|
||||||
|
mysqlgo=true
|
||||||
|
fi
|
||||||
|
#echo "mysqlgo=${mysqlgo}"
|
||||||
|
if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
|
||||||
|
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
|
||||||
|
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||||
fi
|
fi
|
||||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime)
|
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||||
|
touch ${FOLDER}/${nip}-${srvc}.down
|
||||||
|
# fi
|
||||||
|
else
|
||||||
|
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
|
||||||
|
if [ $errtime -gt ${RENOTIFY} ]; then
|
||||||
|
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
|
||||||
|
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
|
||||||
|
fi
|
||||||
|
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
||||||
fi
|
fi
|
||||||
touch ${FOLDER}/${nip}-${docker}.down
|
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||||
|
touch ${FOLDER}/${nip}-${srvc}.down
|
||||||
|
if [ "${srvc}" = "mysql" ]; then
|
||||||
|
${NCMD} systemctl restart ${srvc}
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
else
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
|
||||||
|
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
|
||||||
|
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
|
||||||
|
else
|
||||||
|
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
|
||||||
|
fi
|
||||||
|
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||||
|
|
||||||
|
if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
|
||||||
|
rm -f ${FOLDER}/*-${srvc}.down
|
||||||
|
rm -f ${FOLDER}/*-${srvc}.errtime
|
||||||
|
else
|
||||||
|
rm -f ${FOLDER}/${nip}-${srvc}.down
|
||||||
|
rm -f ${FOLDER}/${nip}-${srvc}.errtime
|
||||||
|
fi
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
else
|
done
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
rm -f ${FOLDER}/${nip}-${docker}.down
|
|
||||||
rm -f ${FOLDER}/${nip}-${docker}.errtime
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
else
|
|
||||||
if [ ! -f ${FOLDER}/${nip}.down ]; then
|
|
||||||
touch ${FOLDER}/${nip}.down
|
|
||||||
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
|
|
||||||
touch ${FOLDER}/${nip}.errtime
|
|
||||||
fi
|
fi
|
||||||
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
|
|
||||||
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
|
if [[ -v ${NTYPE}_DOCKER ]]; then
|
||||||
# fi
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo
|
||||||
|
echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
|
||||||
|
echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
for docker in "${!dockers}"; do
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
c=0; cw=30; spc=''
|
||||||
|
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
|
||||||
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
|
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
|
||||||
|
fi
|
||||||
|
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||||
|
# if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
|
||||||
|
# ${NCMD} docker rm ${docker} >/dev/null 2>&1
|
||||||
|
# fi
|
||||||
|
# ${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
|
||||||
|
# sleep 10s
|
||||||
|
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is fixed\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||||
|
rm -f ${FOLDER}/${nip}-${docker}.down
|
||||||
|
rm -f ${FOLDER}/${nip}-${docker}.errtime
|
||||||
|
else
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
|
||||||
|
|
||||||
|
if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
|
||||||
|
if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo -e "${NM_DOCKERS[${docker}]} is down\t$(date)" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi
|
||||||
|
touch ${FOLDER}/${nip}-${docker}.down
|
||||||
|
else
|
||||||
|
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down)
|
||||||
|
if [ $errtime -gt ${RENOTIFY} ]; then
|
||||||
|
if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then
|
||||||
|
mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime
|
||||||
|
fi
|
||||||
|
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime)
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
touch ${FOLDER}/${nip}-${docker}.down
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
rm -f ${FOLDER}/${nip}-${docker}.down
|
||||||
|
rm -f ${FOLDER}/${nip}-${docker}.errtime
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
else
|
else
|
||||||
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
|
if [ ! -f ${FOLDER}/${nip}.down ]; then
|
||||||
# if [ $errtime -gt ${RENOTIFY} ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then
|
touch ${FOLDER}/${nip}.down
|
||||||
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
|
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
|
||||||
# fi
|
touch ${FOLDER}/${nip}.errtime
|
||||||
|
fi
|
||||||
|
# if [ "${STATUS_CHECK_EMAIL}" != "" ]; then
|
||||||
|
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
|
||||||
|
# fi
|
||||||
|
else
|
||||||
|
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
|
||||||
|
# if [ $errtime -gt ${RENOTIFY} ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then
|
||||||
|
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}
|
||||||
|
# fi
|
||||||
|
fi
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
|
||||||
|
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
if [ "${ST_ACTION}" != "check" ]; then echo; fi
|
||||||
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
|
nid=`expr $nid + 1`
|
||||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
done
|
||||||
fi
|
|
||||||
fi
|
else #GOFORCHECK
|
||||||
if [ "${ST_ACTION}" != "check" ]; then echo; fi
|
|
||||||
nid=`expr $nid + 1`
|
fi #GOFORCHECK
|
||||||
done
|
|
||||||
|
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user