diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 2eba8f1b..cb5da76e 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -174,11 +174,82 @@ STATUS(){ fi if [ "${ST_ACTION}" != "dockers" ]; then - srvcs=${NTYPE}_SERVICES_CHECK[@] - srvcs=${!srvcs} + srvcs=${NTYPE}_SERVICES_CHECK[@]; srvcs=${!srvcs} srvcstotest="$(join_by " " ${srvcs})" - echo "$nip - $srvcstotest" + [ "${srvc}" == "gitea" ] && [ "${NTYPE}" == "WEB" ] && [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *${WEB_HOSTS[0]}* ]] && NOGOCHK=false || NOGOCHK=true + if [ ${NOGOCHK} == true ]; then + if [ "${ST_ACTION}" != "check" ]; then + if [ "${ST_ACTION}" != "report" ]; then + c=0; cw=30; spc=''; spc1=${cw}-${#NM_SERVICES[${srvc}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + else + spc=' ' + fi + echo -en " ${NM_SERVICES[${srvc}]}$spc: " + fi + + srvctst=(`${NCMD} systemctl is-active ${srvcstotest}`) + + sr=0 + for srvc in "${srvcs}"; do + if [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" = "mysql" ]; then + mysqlgo=false; + [ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true; + elif [ "${srvctst[$sr]}" != "active" ] && [ "${srvc}" = "mariadb" ]; then + mysqlgo=false; + [ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true; + elif [ "${srvctst[$sr]}" == "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then + mysqlgo=true + fi + # echo "mysqlgo=${mysqlgo}" + if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + fi + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1 + touch ${FOLDER}/${nip}-${srvc}.down + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE} + + else + errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + if [ $errtime -gt ${RENOTIFY} ]; then + if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime + fi + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + touch ${FOLDER}/${nip}-${srvc}.down + if [ "${srvc}" = "mysql" ]; then + ${NCMD} systemctl restart ${srvc} + fi + fi + [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + fi + else + [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" + if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then + if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) + else + toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) + fi + SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" + echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} + + rm -f ${FOLDER}/${nip}-${srvc}.down + rm -f ${FOLDER}/${nip}-${srvc}.errtime + + fi + # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then + if [ "${srvc}" == "gitea" ]; then + rm -f ${FOLDER}/*-${srvc}.down + rm -f ${FOLDER}/*-${srvc}.errtime + fi + fi + [ "${mysqlgo}" = "true" ] && unset mysqlgo + sr=`expr $sr + 1` + done + fi