update
This commit is contained in:
@@ -133,326 +133,7 @@ STATUSNEW(){
|
|||||||
|
|
||||||
for NTYPE in "${ntypesel[@]}"; do
|
for NTYPE in "${ntypesel[@]}"; do
|
||||||
|
|
||||||
dockers=${NTYPE}_DOCKERS_CHECK[@]
|
STATUS_NODE
|
||||||
hosts=${NTYPE}_HOSTS[@]
|
|
||||||
|
|
||||||
var=${NTYPE}_HOSTS[@]
|
|
||||||
|
|
||||||
if ([[ ! -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NTYPE}]}" == "" ] && [ "${ST_ACTION}" == "services" ]); then
|
|
||||||
GOFORCHECK=false;
|
|
||||||
else
|
|
||||||
GOFORCHECK=true;
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ${GOFORCHECK} = true ]; then
|
|
||||||
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsST[Bold]}"; DIVIDER
|
|
||||||
if [ "${ST_ACTION}" == "dockers" ]; then
|
|
||||||
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
|
|
||||||
else
|
|
||||||
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
DIVIDER . lightYellow; echo -e "${idsST[Reset]}"
|
|
||||||
fi
|
|
||||||
for nip in "${!var}"; do
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}"
|
|
||||||
[ "${ST_ACTION}" != "report" ] && echo -en " - Verifying Host..."
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
|
|
||||||
NCMD=''; LH='- localhost'
|
|
||||||
else
|
|
||||||
NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH=''
|
|
||||||
fi
|
|
||||||
if [ "${NCMD}" != "" ]; then
|
|
||||||
checkhost=$(CHECK_HOST ${nip})
|
|
||||||
[ "${checkhost}" != "false" ] && checkhostssl=$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1)
|
|
||||||
else
|
|
||||||
checkhost=true
|
|
||||||
checkhostssl=ok
|
|
||||||
fi
|
|
||||||
# echo "HERE: $checkhost - $checkhostssl"
|
|
||||||
if [ "${checkhost}" != "false" ] && [ "${checkhostssl}" == "ok" ]; then
|
|
||||||
[ "${NTYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage=""
|
|
||||||
|
|
||||||
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then
|
|
||||||
if [ "${NTYPE}" == "OFW" ]; then
|
|
||||||
uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1)
|
|
||||||
if [ ${#uptime} -ge 6 ]; then
|
|
||||||
uptime="up $(echo $uptime | cut -d: -f1) days, $(echo $uptime | cut -d: -f2) hours, $(echo $uptime | cut -d: -f3) minutes"
|
|
||||||
elif [ ${#uptime} -ge 3 ]; then
|
|
||||||
uptime="up $(echo $uptime | cut -d: -f1) hours, $(echo $uptime | cut -d: -f2) minutes"
|
|
||||||
else
|
|
||||||
uptime="up ${uptime} minutes"
|
|
||||||
fi
|
|
||||||
|
|
||||||
else
|
|
||||||
uptime=$(${NCMD} uptime -p)
|
|
||||||
fi
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[White]} - ${uptime} - ${idsCL[Yellow]}CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%${idsCL[Default]}"
|
|
||||||
else
|
|
||||||
echo -e "${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
DIVIDER false lightCyan
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
|
||||||
if [ -f ${NM_TMPFOLDER}/${nip}.sent ]; then
|
|
||||||
[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
|
|
||||||
fi
|
|
||||||
rm -f ${NM_TMPFOLDER}/${nip}.*
|
|
||||||
fi
|
|
||||||
|
|
||||||
########################
|
|
||||||
## SERVICES CHECK
|
|
||||||
########################
|
|
||||||
if [ "${ST_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NTYPE}]}" != "" ]; then
|
|
||||||
echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}"
|
|
||||||
DIVIDER . green 52
|
|
||||||
|
|
||||||
srvcs=${NTYPE}_SERVICES_CHECK[@];
|
|
||||||
srvcstotest="$(join_by " " ${!srvcs})"
|
|
||||||
[ "${NTYPE}" != "OFW" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}"`)
|
|
||||||
sr=0
|
|
||||||
for srvc in "${!srvcs}"; do
|
|
||||||
if [ "${ST_ACTION}" != "report" ]; then
|
|
||||||
c=0; cw=${PRI_CW}; spc=''
|
|
||||||
spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
|
|
||||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
||||||
else
|
|
||||||
spc=' '
|
|
||||||
fi
|
|
||||||
tmp=${NM_SINGLESRVR_SERVICES[${NTYPE}]}
|
|
||||||
if [[ ${tmp[@]} =~ ${srvc} ]]; then
|
|
||||||
if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NTYPE}]}* ]]; then
|
|
||||||
SSCHK=false
|
|
||||||
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
|
|
||||||
else
|
|
||||||
SSCHK=primary
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
SSCHK=true
|
|
||||||
fi
|
|
||||||
|
|
||||||
[ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]) && SSCHK=false
|
|
||||||
|
|
||||||
if [ ${SSCHK} != false ]; then
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
|
|
||||||
if [ "${NTYPE}" != "OFW" ]; then
|
|
||||||
srvctest=${srvctst[$sr]}
|
|
||||||
if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then
|
|
||||||
[ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false
|
|
||||||
elif [ "${srvctest}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
|
|
||||||
[ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false
|
|
||||||
elif [ "${srvctest}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then
|
|
||||||
mysqlgo=true
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
if [ "$(ssh -o ConnectTimeout=3 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
|
|
||||||
srvctest=active
|
|
||||||
else
|
|
||||||
srvctest=notactive
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
|
|
||||||
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running"
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
|
|
||||||
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
|
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
|
|
||||||
[ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} &
|
|
||||||
|
|
||||||
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then
|
|
||||||
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
|
|
||||||
if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
|
|
||||||
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
|
|
||||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime))
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
|
|
||||||
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
|
|
||||||
|
|
||||||
elif [ ${errtime} -gt 60 ] && [ ${errtime} -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
|
|
||||||
|
|
||||||
fi
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
|
|
||||||
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
|
|
||||||
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
|
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
|
|
||||||
|
|
||||||
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
|
|
||||||
|
|
||||||
fi
|
|
||||||
# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
|
|
||||||
if [ "${srvc}" == "gitea" ] || [ "${srvc}" == "headscale" ]; then
|
|
||||||
rm -f ${NM_FOLDER}/*~${srvc}.*
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
[ "${mysqlgo}" = "true" ] && unset mysqlgo
|
|
||||||
[ "${SSCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}"
|
|
||||||
|
|
||||||
elif [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]); then
|
|
||||||
echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled${idsCL[Default]}"
|
|
||||||
|
|
||||||
else
|
|
||||||
echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}"
|
|
||||||
|
|
||||||
fi
|
|
||||||
sr=`expr $sr + 1`
|
|
||||||
done
|
|
||||||
|
|
||||||
fi
|
|
||||||
|
|
||||||
########################
|
|
||||||
## DOCKER CHECK
|
|
||||||
########################
|
|
||||||
if [[ -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" != "services" ]; then
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
[ "${ST_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NTYPE}]}" != "" ] && echo
|
|
||||||
echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}"
|
|
||||||
DIVIDER . green 52
|
|
||||||
fi
|
|
||||||
for docker in "${!dockers}"; do
|
|
||||||
if [ "${ST_ACTION}" != "report" ]; then
|
|
||||||
c=0; cw=${PRI_CW}; spc=''
|
|
||||||
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
|
|
||||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
||||||
else
|
|
||||||
spc=' '
|
|
||||||
fi
|
|
||||||
tmp=${NM_SINGLESRVR_DOCKERS[${NTYPE}]}
|
|
||||||
if [[ ${tmp[@]} =~ ${docker} ]]; then
|
|
||||||
if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NTYPE}]}* ]]; then
|
|
||||||
SDCHK=false
|
|
||||||
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
|
|
||||||
else
|
|
||||||
SDCHK=primary
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
SDCHK=true
|
|
||||||
fi
|
|
||||||
if [ ${SDCHK} != false ]; then
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
|
|
||||||
|
|
||||||
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
|
||||||
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
|
|
||||||
${NCMD} docker start ${docker} >/dev/null 2>&1
|
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
|
|
||||||
else
|
|
||||||
${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
|
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
|
|
||||||
fi
|
|
||||||
sleep 10s
|
|
||||||
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -en "${idsCL[Green]}Running - Fixed"
|
|
||||||
fi
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed"
|
|
||||||
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
|
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
|
|
||||||
|
|
||||||
else
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
|
|
||||||
|
|
||||||
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
|
|
||||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
|
|
||||||
else
|
|
||||||
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down))
|
|
||||||
if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
|
|
||||||
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then
|
|
||||||
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
|
|
||||||
fi
|
|
||||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
echo -en "${idsCL[LightGreen]}Running"
|
|
||||||
fi
|
|
||||||
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
|
|
||||||
fi
|
|
||||||
[ "${SDCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}"
|
|
||||||
else
|
|
||||||
echo -e "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
else
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}"
|
|
||||||
rm -f ${NM_TMPFOLDER}/${nip}~*
|
|
||||||
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}.down
|
|
||||||
# [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && touch ${NM_TMPFOLDER}/${nip}.errtime
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
|
|
||||||
if [ "${checkhost}" != "false" ]; then
|
|
||||||
echo -e "${idsCL[Red]} - Node is online, but SSL is down!${idsCL[Default]}"
|
|
||||||
else
|
|
||||||
echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
|
|
||||||
[ "${checkhostssl}" != "ok" ] && echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
|
|
||||||
else
|
|
||||||
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
|
|
||||||
mv touch ${NM_TMPFOLDER}/${nip}.down touch ${NM_TMPFOLDER}/${nip}.errtime
|
|
||||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
|
|
||||||
errtime=${toterrtime}
|
|
||||||
if [ "${checkhost}" != "false" ]; then
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is online, but SSL is down" 1
|
|
||||||
else
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
|
|
||||||
fi
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}.sent
|
|
||||||
else
|
|
||||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
|
|
||||||
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "${ST_ACTION}" != "check" ]; then
|
|
||||||
if [ "${checkhost}" != "false" ]; then
|
|
||||||
echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
|
||||||
else
|
|
||||||
echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
if [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then
|
|
||||||
if [ "${checkhost}" != "false" ]; then
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSL has been down for $(SHOW_TIME ${toterrtime})" 1
|
|
||||||
else
|
|
||||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
|
|
||||||
fi
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}.sent
|
|
||||||
fi
|
|
||||||
touch ${NM_TMPFOLDER}/${nip}.down
|
|
||||||
fi
|
|
||||||
|
|
||||||
fi
|
|
||||||
[ "${ST_ACTION}" != "check" ] && echo
|
|
||||||
done
|
|
||||||
|
|
||||||
fi #GOFORCHECK
|
|
||||||
|
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -737,3 +418,326 @@ It was down for $(SHOW_TIME ${toterrtime})"
|
|||||||
echo -e "Runtime: ${runtime}\n"
|
echo -e "Runtime: ${runtime}\n"
|
||||||
[ "${ST_ACTION}" == "report" ] && echo ${runtime} >> ${NM_LOGFOLDER}/status-check.scantimes
|
[ "${ST_ACTION}" == "report" ] && echo ${runtime} >> ${NM_LOGFOLDER}/status-check.scantimes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
STATUS_NODE(){
|
||||||
|
dockers=${NTYPE}_DOCKERS_CHECK[@]
|
||||||
|
hosts=${NTYPE}_HOSTS[@]
|
||||||
|
|
||||||
|
var=${NTYPE}_HOSTS[@]
|
||||||
|
|
||||||
|
if ([[ ! -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NTYPE}]}" == "" ] && [ "${ST_ACTION}" == "services" ]); then
|
||||||
|
GOFORCHECK=false;
|
||||||
|
else
|
||||||
|
GOFORCHECK=true;
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${GOFORCHECK} = true ]; then
|
||||||
|
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsST[Bold]}"; DIVIDER
|
||||||
|
if [ "${ST_ACTION}" == "dockers" ]; then
|
||||||
|
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
|
||||||
|
else
|
||||||
|
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
DIVIDER . lightYellow; echo -e "${idsST[Reset]}"
|
||||||
|
fi
|
||||||
|
for nip in "${!var}"; do
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}"
|
||||||
|
[ "${ST_ACTION}" != "report" ] && echo -en " - Verifying Host..."
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
|
||||||
|
NCMD=''; LH='- localhost'
|
||||||
|
else
|
||||||
|
NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH=''
|
||||||
|
fi
|
||||||
|
if [ "${NCMD}" != "" ]; then
|
||||||
|
checkhost=$(CHECK_HOST ${nip})
|
||||||
|
[ "${checkhost}" != "false" ] && checkhostssl=$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1)
|
||||||
|
else
|
||||||
|
checkhost=true
|
||||||
|
checkhostssl=ok
|
||||||
|
fi
|
||||||
|
# echo "HERE: $checkhost - $checkhostssl"
|
||||||
|
if [ "${checkhost}" != "false" ] && [ "${checkhostssl}" == "ok" ]; then
|
||||||
|
[ "${NTYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage=""
|
||||||
|
|
||||||
|
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then
|
||||||
|
if [ "${NTYPE}" == "OFW" ]; then
|
||||||
|
uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1)
|
||||||
|
if [ ${#uptime} -ge 6 ]; then
|
||||||
|
uptime="up $(echo $uptime | cut -d: -f1) days, $(echo $uptime | cut -d: -f2) hours, $(echo $uptime | cut -d: -f3) minutes"
|
||||||
|
elif [ ${#uptime} -ge 3 ]; then
|
||||||
|
uptime="up $(echo $uptime | cut -d: -f1) hours, $(echo $uptime | cut -d: -f2) minutes"
|
||||||
|
else
|
||||||
|
uptime="up ${uptime} minutes"
|
||||||
|
fi
|
||||||
|
|
||||||
|
else
|
||||||
|
uptime=$(${NCMD} uptime -p)
|
||||||
|
fi
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[White]} - ${uptime} - ${idsCL[Yellow]}CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%${idsCL[Default]}"
|
||||||
|
else
|
||||||
|
echo -e "${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DIVIDER false lightCyan
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
||||||
|
if [ -f ${NM_TMPFOLDER}/${nip}.sent ]; then
|
||||||
|
[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
|
||||||
|
fi
|
||||||
|
rm -f ${NM_TMPFOLDER}/${nip}.*
|
||||||
|
fi
|
||||||
|
|
||||||
|
########################
|
||||||
|
## SERVICES CHECK
|
||||||
|
########################
|
||||||
|
if [ "${ST_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NTYPE}]}" != "" ]; then
|
||||||
|
echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}"
|
||||||
|
DIVIDER . green 52
|
||||||
|
|
||||||
|
srvcs=${NTYPE}_SERVICES_CHECK[@];
|
||||||
|
srvcstotest="$(join_by " " ${!srvcs})"
|
||||||
|
[ "${NTYPE}" != "OFW" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}"`)
|
||||||
|
sr=0
|
||||||
|
for srvc in "${!srvcs}"; do
|
||||||
|
if [ "${ST_ACTION}" != "report" ]; then
|
||||||
|
c=0; cw=${PRI_CW}; spc=''
|
||||||
|
spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
|
||||||
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
|
else
|
||||||
|
spc=' '
|
||||||
|
fi
|
||||||
|
tmp=${NM_SINGLESRVR_SERVICES[${NTYPE}]}
|
||||||
|
if [[ ${tmp[@]} =~ ${srvc} ]]; then
|
||||||
|
if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NTYPE}]}* ]]; then
|
||||||
|
SSCHK=false
|
||||||
|
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
|
||||||
|
else
|
||||||
|
SSCHK=primary
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
SSCHK=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
[ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]) && SSCHK=false
|
||||||
|
|
||||||
|
if [ ${SSCHK} != false ]; then
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
|
||||||
|
if [ "${NTYPE}" != "OFW" ]; then
|
||||||
|
srvctest=${srvctst[$sr]}
|
||||||
|
if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then
|
||||||
|
[ "$(${NCMD} systemctl is-active mariadb)" == "active" ] && mysqlgo=true || mysqlgo=false
|
||||||
|
elif [ "${srvctest}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
|
||||||
|
[ "$(${NCMD} systemctl is-active mysql)" == "active" ] && mysqlgo=true || mysqlgo=false
|
||||||
|
elif [ "${srvctest}" == "active" ] && ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]); then
|
||||||
|
mysqlgo=true
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if [ "$(ssh -o ConnectTimeout=3 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
|
||||||
|
srvctest=active
|
||||||
|
else
|
||||||
|
srvctest=notactive
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
|
||||||
|
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running"
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
|
||||||
|
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
|
||||||
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
|
||||||
|
[ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} &
|
||||||
|
|
||||||
|
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then
|
||||||
|
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
|
||||||
|
if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
|
||||||
|
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
|
||||||
|
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime))
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
|
||||||
|
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
|
||||||
|
|
||||||
|
elif [ ${errtime} -gt 60 ] && [ ${errtime} -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
|
||||||
|
|
||||||
|
fi
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
|
||||||
|
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
|
||||||
|
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
|
||||||
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
|
||||||
|
|
||||||
|
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
|
||||||
|
|
||||||
|
fi
|
||||||
|
# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
|
||||||
|
if [ "${srvc}" == "gitea" ] || [ "${srvc}" == "headscale" ]; then
|
||||||
|
rm -f ${NM_FOLDER}/*~${srvc}.*
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
[ "${mysqlgo}" = "true" ] && unset mysqlgo
|
||||||
|
[ "${SSCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}"
|
||||||
|
|
||||||
|
elif [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]); then
|
||||||
|
echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled${idsCL[Default]}"
|
||||||
|
|
||||||
|
else
|
||||||
|
echo -e "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}"
|
||||||
|
|
||||||
|
fi
|
||||||
|
sr=`expr $sr + 1`
|
||||||
|
done
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
########################
|
||||||
|
## DOCKER CHECK
|
||||||
|
########################
|
||||||
|
if [[ -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" != "services" ]; then
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
[ "${ST_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NTYPE}]}" != "" ] && echo
|
||||||
|
echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}"
|
||||||
|
DIVIDER . green 52
|
||||||
|
fi
|
||||||
|
for docker in "${!dockers}"; do
|
||||||
|
if [ "${ST_ACTION}" != "report" ]; then
|
||||||
|
c=0; cw=${PRI_CW}; spc=''
|
||||||
|
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
|
||||||
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||||
|
else
|
||||||
|
spc=' '
|
||||||
|
fi
|
||||||
|
tmp=${NM_SINGLESRVR_DOCKERS[${NTYPE}]}
|
||||||
|
if [[ ${tmp[@]} =~ ${docker} ]]; then
|
||||||
|
if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NTYPE}]}* ]]; then
|
||||||
|
SDCHK=false
|
||||||
|
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
|
||||||
|
else
|
||||||
|
SDCHK=primary
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
SDCHK=true
|
||||||
|
fi
|
||||||
|
if [ ${SDCHK} != false ]; then
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
|
||||||
|
|
||||||
|
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||||
|
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
|
||||||
|
${NCMD} docker start ${docker} >/dev/null 2>&1
|
||||||
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
|
||||||
|
else
|
||||||
|
${NCMD} docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
|
||||||
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
|
||||||
|
fi
|
||||||
|
sleep 10s
|
||||||
|
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -en "${idsCL[Green]}Running - Fixed"
|
||||||
|
fi
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed"
|
||||||
|
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
|
||||||
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
|
||||||
|
|
||||||
|
else
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
|
||||||
|
|
||||||
|
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
|
||||||
|
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
|
||||||
|
else
|
||||||
|
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down))
|
||||||
|
if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
|
||||||
|
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then
|
||||||
|
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
|
||||||
|
fi
|
||||||
|
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
echo -en "${idsCL[LightGreen]}Running"
|
||||||
|
fi
|
||||||
|
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
|
||||||
|
fi
|
||||||
|
[ "${SDCHK}" == "primary" ] && echo -e "${idsCL[LightCyan]} - Primary Node${idsCL[Default]}" || echo -e "${idsCL[Default]}"
|
||||||
|
else
|
||||||
|
echo -e "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}${spc}${idsCL[Default]}: ${idsCL[LightCyan]}Disabled - Secondary Node${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
else
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}"
|
||||||
|
rm -f ${NM_TMPFOLDER}/${nip}~*
|
||||||
|
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}.down
|
||||||
|
# [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && touch ${NM_TMPFOLDER}/${nip}.errtime
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
|
||||||
|
if [ "${checkhost}" != "false" ]; then
|
||||||
|
echo -e "${idsCL[Red]} - Node is online, but SSL is down!${idsCL[Default]}"
|
||||||
|
else
|
||||||
|
echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
|
||||||
|
[ "${checkhostssl}" != "ok" ] && echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
|
||||||
|
else
|
||||||
|
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
|
||||||
|
mv touch ${NM_TMPFOLDER}/${nip}.down touch ${NM_TMPFOLDER}/${nip}.errtime
|
||||||
|
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
|
||||||
|
errtime=${toterrtime}
|
||||||
|
if [ "${checkhost}" != "false" ]; then
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is online, but SSL is down" 1
|
||||||
|
else
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
|
||||||
|
fi
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}.sent
|
||||||
|
else
|
||||||
|
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
|
||||||
|
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "${ST_ACTION}" != "check" ]; then
|
||||||
|
if [ "${checkhost}" != "false" ]; then
|
||||||
|
echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
||||||
|
else
|
||||||
|
echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then
|
||||||
|
if [ "${checkhost}" != "false" ]; then
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSL has been down for $(SHOW_TIME ${toterrtime})" 1
|
||||||
|
else
|
||||||
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
|
||||||
|
fi
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}.sent
|
||||||
|
fi
|
||||||
|
touch ${NM_TMPFOLDER}/${nip}.down
|
||||||
|
fi
|
||||||
|
|
||||||
|
fi
|
||||||
|
[ "${ST_ACTION}" != "check" ] && echo
|
||||||
|
done
|
||||||
|
|
||||||
|
fi #GOFORCHECK
|
||||||
|
}
|
||||||
|
|||||||
@@ -864,6 +864,9 @@ GUI(){
|
|||||||
dl)
|
dl)
|
||||||
DOWNLIST ${2}
|
DOWNLIST ${2}
|
||||||
;;
|
;;
|
||||||
|
statusnew)
|
||||||
|
STATUSNEW
|
||||||
|
;;
|
||||||
status)
|
status)
|
||||||
if [ "${2}" == "freespace" ]; then
|
if [ "${2}" == "freespace" ]; then
|
||||||
STATUS freespace ${3}
|
STATUS freespace ${3}
|
||||||
|
|||||||
Reference in New Issue
Block a user